RemoveDuplicates.org

{'type': 'object', 'required': ['text'], 'properties': {'keep': {'enum': ['first', 'last'], 'type': 'string', 'default': 'first', 'description': 'Which occurrence of a duplicate survives.'}, 'text': {'type': 'string', 'description': 'The list or table text: one item per line, or CSV/tab-separated rows. Up to 131072 UTF-8 bytes.'}, 'trim': {'type': 'boolean', 'default': False, 'description': 'Ignore surrounding spaces when comparing and return trimmed values.'}, 'order': {'enum': ['preserve', 'sort'], 'type': 'string', 'default': 'preserve', 'description': 'preserve keeps the original order; sort returns a natural (numeric-aware) sort.'}, 'format': {'enum': ['auto', 'lines', 'table'], 'type': 'string', 'default': 'auto', 'description': 'auto detects CSV/TSV tables; lines compares whole lines; table forces CSV/TSV parsing.'}, 'header': {'type': 'boolean', 'default': False, 'description': 'Tables only: keep the first row as a header outside duplicate checks and counts.'}, 'compare': {'anyOf': [{'enum': ['row'], 'type': 'string'}, {'type': 'integer', 'minimum': 0}, {'type': 'array', 'items': {'type': 'integer', 'minimum': 0}, 'minItems': 1, 'uniqueItems': True}], 'default': 'row', 'description': 'Tables only: "row" compares entire rows; a zero-based column index or an array of indices compares only those columns.'}, 'ignoreCase': {'type': 'boolean', 'default': False, 'description': 'Treat values such as Apple and apple as duplicates.'}, 'removeEmpty': {'type': 'boolean', 'default': True, 'description': 'Drop blank lines or all-blank rows.'}}, 'additionalProperties': False}

{'type': 'object', 'required': ['text', 'format', 'stats'], 'properties': {'text': {'type': 'string', 'description': 'The cleaned text, rows joined with \\n.'}, 'stats': {'type': 'object', 'required': ['total', 'unique', 'removed', 'duplicates', 'emptyRemoved', 'reduction'], 'properties': {'total': {'type': 'integer', 'description': 'Input lines or data rows (header excluded).'}, 'unique': {'type': 'integer', 'description': 'Lines or data rows kept.'}, 'removed': {'type': 'integer', 'description': 'duplicates + emptyRemoved.'}, 'reduction': {'type': 'number', 'description': 'Percent of input removed, one decimal.'}, 'duplicates': {'type': 'integer'}, 'emptyRemoved': {'type': 'integer'}}}, 'format': {'type': 'object', 'required': ['kind', 'columns', 'header'], 'properties': {'kind': {'enum': ['lines', 'csv', 'tsv'], 'type': 'string'}, 'header': {'type': 'boolean'}, 'columns': {'type': 'integer'}}}}}