How to use pyhindsight - 10 common examples

w.write(row_number, 18, item.server_name, gray_value_format)  # Server name
                    w.write(row_number, 19, item.location, gray_value_format)  # Cached data location // data_2 [1542523]
                    w.write(row_number, 20, item.http_headers_str, gray_value_format)  # Cached data location // data_2 [1542523]

                elif item.row_type.startswith("local storage"):
                    w.write_string(row_number, 0, item.row_type, gray_type_format)  # record_type
                    w.write(row_number, 1, friendly_date(item.timestamp), gray_date_format)  # date
                    w.write_string(row_number, 2, item.url, gray_url_format)  # URL
                    w.write_string(row_number, 3, item.name, gray_field_format)  # cookie name
                    w.write_string(row_number, 4, item.value, gray_value_format)  # cookie value
                    w.write(row_number, 5, item.interpretation, gray_value_format)  # cookie interpretation
                    w.write(row_number, 6, item.profile, gray_value_format)  # Profile

                elif item.row_type.startswith("login"):
                    w.write_string(row_number, 0, item.row_type, red_type_format)  # record_type
                    w.write(row_number, 1, friendly_date(item.timestamp), red_date_format)  # date
                    w.write_string(row_number, 2, item.url, red_url_format)  # URL
                    w.write_string(row_number, 3, item.name, red_field_format)  # form field name
                    w.write_string(row_number, 4, item.value, red_value_format)  # username or pw value
                    w.write(row_number, 6, item.profile, red_value_format)  # Profile

                elif item.row_type.startswith("preference"):
                    w.write_string(row_number, 0, item.row_type, blue_type_format)  # record_type
                    w.write(row_number, 1, friendly_date(item.timestamp), blue_date_format)  # date
                    w.write_string(row_number, 2, item.url, blue_url_format)  # URL
                    w.write_string(row_number, 3, item.name, blue_field_format)  # form field name
                    w.write_string(row_number, 4, item.value, blue_value_format)  # username or pw value
                    w.write(row_number, 5, item.interpretation, blue_value_format)  # interpretation
                    w.write(row_number, 6, item.profile, blue_value_format)  # Profile

            except Exception, e:
                log.error("Failed to write row to XLSX: {}".format(e))

if item.name == u'__utmc':
                m = re.search(utmc_re, item.value)
                if m:
                    item.interpretation = u'Domain Hash: {} | [Google Analytics Cookie]'.format(m.group(1))
                    parsedItems += 1
            if item.name == u'__utmv':
                m = re.search(utmv_re, item.value)
                if m:
                    item.interpretation = u'Domain Hash: {} | Custom Values: {} | [Google Analytics Cookie]' \
                                          .format(m.group(1), urllib.unquote_plus(m.group(2)))
                    parsedItems += 1
            if item.name == u'__utmz':
                m = re.search(utmz_re, item.value)
                if m:
                    derived = u'Domain Hash: {} | Last Visit: {} | Sessions: {} | Sources: {} | ' \
                              .format(m.group(1), friendly_date(m.group(2)), m.group(3), m.group(4))
                    parsedItems += 1
                    p = re.search(utmz_parameters_re, item.value)

                    parameters = {}
                    raw_parameters = p.group(5)[3:]  # Strip off first 'utm' so later splitting will work
                    # print(raw_parameters)

                    #Parse out cookie fields
                    for pair in raw_parameters.split(u'|utm'):               # Split the cookie on the '|' delimiter
                        # print pair
                        rp = re.search(utmz_extract_parameters_re, pair)    # Split each parameter on the first '='
                        try:
                            parameters[rp.group(1)] = rp.group(2)           # Put the parameter name and value in hash
                        except AttributeError:
                            pass
                    if u'cmd' in parameters:

elif re.search(rename_re, user_input):
                    output_file = "{}_1.sqlite".format(output_file[:-7])
                    print("Renaming new output to {}".format(output_file))
                else:
                    print("Did not understand response.  Exiting... ")
                    sys.exit()

        analysis_session.generate_sqlite(output_file)

    print(banner)

    # Useful when Hindsight is run from a different directory than where the file is located
    real_path = os.path.dirname(os.path.realpath(sys.argv[0]))

    # Set up the AnalysisSession object, and transfer the relevant input arguments to it
    analysis_session = AnalysisSession()

    # parse_arguments needs the analysis_session as an input to set things like available decrypts
    args = parse_arguments(analysis_session)

    if args.output:
        analysis_session.output_name = args.output

    if args.cache:
        analysis_session.cache_path = args.cache

    analysis_session.selected_output_format = args.format
    analysis_session.browser_type = args.browser_type
    analysis_session.timezone = args.timezone

    if args.log == 'hindsight.log':
        args.log = os.path.join(real_path, args.log)

log.debug(" - Loading '{}' [custom plugin]".format(plugin))
                                    try:
                                        module = __import__(plugin)
                                    except ImportError, e:
                                        log.error(" - Error: {}".format(e))
                                        print format_plugin_output(plugin, "-unknown", 'import failed (see log)')
                                        continue
                                    try:
                                        log.info(" - Running '{}' plugin".format(module.friendlyName))
                                        parsed_items = module.plugin(self)
                                        print format_plugin_output(module.friendlyName, module.version, parsed_items)
                                        self.plugin_results[plugin] = [module.friendlyName, module.version, parsed_items]
                                        log.info(" - Completed; {}".format(parsed_items))
                                        completed_plugins.append(plugin)
                                    except Exception, e:
                                        print format_plugin_output(module.friendlyName, module.version, 'failed')
                                        self.plugin_results[plugin] = [module.friendlyName, module.version, 'failed']
                                        log.info(" - Failed; {}".format(e))
                    except Exception as e:
                        log.debug(' - Error loading plugins ({})'.format(e))
                        print '  - Error loading plugins'
                    finally:
                        # Remove the current plugin location from the system path, so we don't loop over it again
                        sys.path.remove(potential_plugin_path)

# Check to see if we've already run this plugin (likely from a different path)
                    if plugin in completed_plugins:
                        log.info(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
                        continue

                    log.info(" - Loading '{}' [standard plugin]".format(plugin))
                    try:
                        module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
                    except ImportError, e:
                        log.error(" - Error: {}".format(e))
                        print format_plugin_output(plugin, "-unknown", 'import failed (see log)')
                        continue
                    try:
                        log.info(" - Running '{}' plugin".format(module.friendlyName))
                        parsed_items = module.plugin(self)
                        print format_plugin_output(module.friendlyName, module.version, parsed_items)
                        self.plugin_results[plugin] = [module.friendlyName, module.version, parsed_items]
                        log.info(" - Completed; {}".format(parsed_items))
                        completed_plugins.append(plugin)
                        break
                    except Exception, e:
                        print format_plugin_output(module.friendlyName, module.version, 'failed')
                        self.plugin_results[plugin] = [module.friendlyName, module.version, 'failed']
                        log.info(" - Failed; {}".format(e))

            for potential_path in sys.path:
                # If a subdirectory exists called 'plugins' at the current path, continue on
                potential_plugin_path = os.path.join(potential_path, 'plugins')
                if os.path.isdir(potential_plugin_path):
                    try:
                        # Insert the current plugin location to the system path, so we can import plugin modules by name
                        sys.path.insert(0, potential_plugin_path)

log.info("Plugins:")
    completed_plugins = []

    # First run built-in plugins that ship with Hindsight
    log.info(" Built-in Plugins:")
    for plugin in pyhindsight.plugins.__all__:
        # Check to see if we've already run this plugin (likely from a different path)
        if plugin in completed_plugins:
            continue

        log.debug(" - Loading '{}'".format(plugin))
        try:
            module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
        except ImportError, e:
            log.error(" - Error: {}".format(e))
            print(format_plugin_output(plugin, "-unknown", 'import failed (see log)'))
            continue
        try:
            log.info(" - Running '{}' plugin".format(module.friendlyName))
            parsed_items = module.plugin(analysis_session)
            print(format_plugin_output(module.friendlyName, module.version, parsed_items))
            log.info(" - Completed; {}".format(parsed_items))
            completed_plugins.append(plugin)
        except Exception, e:
            print(format_plugin_output(module.friendlyName, module.version, 'failed'))
            log.info(" - Failed; {}".format(e))

    # Then look for any custom user-provided plugins in a 'plugins' directory
    log.info(" Custom Plugins:")

    if real_path not in sys.path:
        sys.path.insert(0, real_path)

log.debug(" - Loading '{}'".format(plugin))
        try:
            module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
        except ImportError, e:
            log.error(" - Error: {}".format(e))
            print(format_plugin_output(plugin, "-unknown", 'import failed (see log)'))
            continue
        try:
            log.info(" - Running '{}' plugin".format(module.friendlyName))
            parsed_items = module.plugin(analysis_session)
            print(format_plugin_output(module.friendlyName, module.version, parsed_items))
            log.info(" - Completed; {}".format(parsed_items))
            completed_plugins.append(plugin)
        except Exception, e:
            print(format_plugin_output(module.friendlyName, module.version, 'failed'))
            log.info(" - Failed; {}".format(e))

    # Then look for any custom user-provided plugins in a 'plugins' directory
    log.info(" Custom Plugins:")

    if real_path not in sys.path:
        sys.path.insert(0, real_path)

    # Loop through all paths, to pick up all potential locations for custom plugins
    for potential_path in sys.path:
        # If a subdirectory exists called 'plugins' or 'pyhindsight/plugins' at the current path, continue on
        for potential_plugin_path in [os.path.join(potential_path, 'plugins'), os.path.join(potential_path, 'pyhindsight', 'plugins')]:
            if os.path.isdir(potential_plugin_path):
                log.info(" Found custom plugin directory {}:".format(potential_plugin_path))
                try:
                    # Insert the current plugin location to the system path, so we can import plugin modules by name

log.debug(" - Loading '{}'".format(plugin))
                            try:
                                module = __import__(plugin)
                            except ImportError, e:
                                log.error(" - Error: {}".format(e))
                                print(format_plugin_output(plugin, "-unknown", 'import failed (see log)'))
                                continue
                            try:
                                log.info(" - Running '{}' plugin".format(module.friendlyName))
                                parsed_items = module.plugin(analysis_session)
                                print(format_plugin_output(module.friendlyName, module.version, parsed_items))
                                log.info(" - Completed; {}".format(parsed_items))
                                completed_plugins.append(plugin)
                            except Exception, e:
                                print(format_plugin_output(module.friendlyName, module.version, 'failed'))
                                log.info(" - Failed; {}".format(e))
                except Exception as e:
                    log.debug(' - Error loading plugins ({})'.format(e))
                    print('  - Error loading plugins')
                finally:
                    # Remove the current plugin location from the system path, so we don't loop over it again
                    sys.path.remove(potential_plugin_path)

    # Check if output directory exists; attempt to create if it doesn't
    if os.path.dirname(analysis_session.output_name) != "" and not os.path.exists(os.path.dirname(analysis_session.output_name)):
        os.makedirs(os.path.dirname(analysis_session.output_name))

    # Get desired output type form args.format and call the correct output creation function
    if analysis_session.selected_output_format == 'xlsx':
        log.info("Writing output; XLSX format selected")
        try:

log.debug(" - Contents of plugin folder: " + str(plugin_listing))
                    for plugin in plugin_listing:
                        if plugin[-3:] == ".py" and plugin[0] != '_':
                            plugin = plugin.replace(".py", "")

                            # Check to see if we've already run this plugin (likely from a different path)
                            if plugin in completed_plugins:
                                log.debug(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
                                continue

                            log.debug(" - Loading '{}'".format(plugin))
                            try:
                                module = __import__(plugin)
                            except ImportError, e:
                                log.error(" - Error: {}".format(e))
                                print(format_plugin_output(plugin, "-unknown", 'import failed (see log)'))
                                continue
                            try:
                                log.info(" - Running '{}' plugin".format(module.friendlyName))
                                parsed_items = module.plugin(analysis_session)
                                print(format_plugin_output(module.friendlyName, module.version, parsed_items))
                                log.info(" - Completed; {}".format(parsed_items))
                                completed_plugins.append(plugin)
                            except Exception, e:
                                print(format_plugin_output(module.friendlyName, module.version, 'failed'))
                                log.info(" - Failed; {}".format(e))
                except Exception as e:
                    log.debug(' - Error loading plugins ({})'.format(e))
                    print('  - Error loading plugins')
                finally:
                    # Remove the current plugin location from the system path, so we don't loop over it again
                    sys.path.remove(potential_plugin_path)

# Check to see if we've already run this plugin (likely from a different path)
                            if plugin in completed_plugins:
                                log.debug(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
                                continue

                            log.debug(" - Loading '{}'".format(plugin))
                            try:
                                module = __import__(plugin)
                            except ImportError, e:
                                log.error(" - Error: {}".format(e))
                                print(format_plugin_output(plugin, "-unknown", 'import failed (see log)'))
                                continue
                            try:
                                log.info(" - Running '{}' plugin".format(module.friendlyName))
                                parsed_items = module.plugin(analysis_session)
                                print(format_plugin_output(module.friendlyName, module.version, parsed_items))
                                log.info(" - Completed; {}".format(parsed_items))
                                completed_plugins.append(plugin)
                            except Exception, e:
                                print(format_plugin_output(module.friendlyName, module.version, 'failed'))
                                log.info(" - Failed; {}".format(e))
                except Exception as e:
                    log.debug(' - Error loading plugins ({})'.format(e))
                    print('  - Error loading plugins')
                finally:
                    # Remove the current plugin location from the system path, so we don't loop over it again
                    sys.path.remove(potential_plugin_path)

    # Check if output directory exists; attempt to create if it doesn't
    if os.path.dirname(analysis_session.output_name) != "" and not os.path.exists(os.path.dirname(analysis_session.output_name)):
        os.makedirs(os.path.dirname(analysis_session.output_name))