How to use the acorn.tokenizer function in acorn

dependency_tree.push(curdep);
	}

	// skip already scanned file
	if (curdep.scanned) return;

	// only scan each dependency once
	curdep.scanned = true;

	// read file contents
	contents = fs.readFileSync(filepath,{ encoding: "utf8" });

	// consume all tokens so comments are extracted
	try {
		// prepare tokenizer for file
		tokenizer = acorn.tokenizer(contents,parse_options);

		do {
			token = tokenizer.getToken();
		}
		while (token && token.type != acorn.tokTypes.eof);
	}
	catch (err) {
		if (!OPTS.ignore.invalid) {
			if (/^Invalid:/.test(err.message)) {
				throw err;
			}
			else {
				throw new Error("Invalid: " + filepath + "\n" + err.toString());
			}
		}
	}

function run(code) {
  let result
  if (!tokenize) {
    try { result = acorn.parse(code, options) }
    catch(e) { console.error(e.message); process.exit(1) }
  } else {
    result = []
    let tokenizer = acorn.tokenizer(code, options), token
    while (true) {
      try { token = tokenizer.getToken() }
      catch(e) { console.error(e.message); process.exit(1) }
      result.push(token)
      if (token.type == acorn.tokTypes.eof) break
    }
  }
  if (!silent) console.log(JSON.stringify(result, null, compact ? null : 2))
}

function run(code) {
  let result
  if (!tokenize) {
    try { result = acorn.parse(code, options) }
    catch(e) { console.error(e.message); process.exit(1) }
  } else {
    result = []
    let tokenizer = acorn.tokenizer(code, options), token
    while (true) {
      try { token = tokenizer.getToken() }
      catch(e) { console.error(e.message); process.exit(1) }
      result.push(token)
      if (token.type == acorn.tokTypes.eof) break
    }
  }
  if (!silent) console.log(JSON.stringify(result, null, compact ? null : 2))
}

function run(code) {
  let result
  try {
    if (!tokenize) {
      result = acorn.parse(code, options)
    } else {
      result = []
      let tokenizer = acorn.tokenizer(code, options), token
      do {
        token = tokenizer.getToken()
        result.push(token)
      } while (token.type != acorn.tokTypes.eof)
    }
  } catch (e) {
    console.error(e.message)
    process.exit(1)
  }
  if (!silent) console.log(JSON.stringify(result, null, compact ? null : 2))
}

function generateMap(content, file, offset) {
  if (offset <= 0) {
    throw new RangeError('Offset parameter should be greater than zero.');
  }

  let generator = new sourceMap.SourceMapGenerator();
  let options = {
    locations: true,
    sourceType: 'module',
  };

  for (let token of tokenizer(content, options)) {
    let position = token.loc.start;

    generator.addMapping({
      source: file,
      original: {line: position.line + offset, column: position.column},
      generated: position,
      name: token.value,
    });
  }

  return generator.toJSON();
}

tokenizer: function tokenizer (src, opts) {
    return acorn.tokenizer(src, mapOptions(opts))
  }
})

function generateJs(sourcePath, fileContent) {
  var generator = new SourceMapGenerator({ file: sourcePath });
  var tokenizer = acorn.tokenizer(fileContent, { locations: true });

  while (true) {
    var token = tokenizer.getToken();

    if (token.type.label === 'eof') {
      break;
    }
    var mapping = {
      original: token.loc.start,
      generated: token.loc.start,
      source: sourcePath
    };
    if (token.type.label === 'name') {
      mapping.name = token.value;
    }
    generator.addMapping(mapping);

tokenizer: function tokenizer (src, opts) {
    return acorn.tokenizer(src, mapOptions(opts))
  }
})