How to use queue - 10 common examples

function ensureVideosForProduct(product, playlist, callback) {
    const q = queue({concurrency: 1});
    // ensures that all videos which exist in outPath for this product are uploaded to Youtube with the correct info
    // get the list of items in the playlist from API
    listPlaylistItems(playlist.id, (err, data) => {
        catchErr(err);
        const playlistItems = data.items;
        console.log(`got ${playlistItems.length} items in playlist`);
        const videoIds = _.pluck(playlistItems, 'snippet.resourceId.videoId');
        console.log(videoIds.length)
        // have to get video information with videos.list because playlistItems doesn't return tags
        getVideoById(videoIds, (err, data) => {
            catchErr(err);
            console.log(`got info for ${data.items.length} videos`);
            // video id:x tags contain the sessionId, ie. directory name, eg. '20150920210000-20150921123000'
            const videosBySessionId = _.omit(_.indexBy(data.items, getIdFromTag), null);
            const sessionDirs = sh.ls(outPath);

function uploadVideosForProducts(products, callback=_.noop) {
    const q = queue({concurrency: 1});
    // get playlists from Youtube API
    listPlaylists((err, data) => {
        catchErr(err);
        // and find their associated product IDs (via the 'id:x' tag)
        const playlistsByProductId = _.omit(_.indexBy(data.items, getIdFromTag), null);
        const sessionDirs = sh.ls(outPath);

        // then ensure videos exist for each product
        products.forEach(product => {
            const productPlaylist = playlistsByProductId[product.id];
            if(!productPlaylist) return; // assume playlists exist
            q.push(function(next) {
                ensureVideosForProduct(product, productPlaylist, () => { 
                    console.log(`all videos uploaded for ${product.title}`);
                    next();
                });

// if already quitting then force quit
    if (quitting) {
      console.log('forcing quit now');
      process.exit();
    }
    await quit();
  });

  try {
    console.log('worker initializing');

    db = new Database(databaseConfig);
    await db.connect();
    console.log('connected to database');

    consumer = new Consumer('queue', queueConfig);
    consumer.on('error', err => {
      console.log(err.message);
      process.exit(1);
    });
    await new Promise(resolve => {
      consumer.on('ready', async() => {
        resolve();
      });
    });
    console.log('connected to queue');

    console.log('worker initialized');
  } catch (err) {
    console.log(err);
    process.exit(1);
  }

process.env.AWS_XRAY_DEBUG_MODE=1;

const Consumer = require('queue').Consumer;
const Database = require('@subfuzion/database').Database;
const xray = require('aws-xray-sdk-core');

// set queue connection timeout to 0 since we want the worker queue
// consumer to block indefinitely while waiting for messages
let queueConfig = Consumer.createStdConfig({ timeout: 0 });
let databaseConfig = Database.createStdConfig();

let consumer, db, quitting = false;

// Set up signal handlers and open connections to database and queue.
async function init() {
  // Handle SIGTERM and SIGINT (ctrl-c) gracefully
  process.on('SIGTERM', async () => {
    console.log('worker received SIGTERM');
    // if already quitting then force quit
    if (quitting) {
      console.log('forcing quit now');
      process.exit();
    }
    await quit();
  });

function ensurePlaylistsForProducts(products, callback, {makeTitle, makeDescription=()=>''}={}) {
    const q = queue({concurrency: 1});
    if(!makeTitle) makeTitle = (product) => product.title;

    // first get playlists from Youtube API
    listPlaylists((err, data) => {
        catchErr(err);
        // and find their associated product IDs (via the 'id:x' tag)
        const playlistsByProductId = _.omit(_.indexBy(data.items, getIdFromTag), null);

        // for each expected product, ensure a playlist exists and has the correct information
        products.forEach(product => {
            const productPlaylist = playlistsByProductId[product.id];
            const updatedPlaylist = {
                title: makeTitle(product),
                description: makeDescription(product),
                tags: [`id:${product.id}`].concat(product.tags || [])
            };

function main(callback) {
    const q = queue({concurrency: 1});
    q.push(
        //cb => retrieveImages(cb),
        // cb => resizeImages(cb),
        cb => {
            makeVideos(`${imgPath}/50%`, `${imgPath}/50%/video`);
            cb();
        },
        // cb => cropImages(crops, cb),
        // cb => makeCroppedVideos(crops, cb),
        
        cb => { console.log('all done!'); cb(); }
    );
    q.start(callback);
}
main();

import {config} from '../../config'
import {broadcast, epoch} from '../both/util'
import {
	indexGenre,
	indexMovieGenre,
	getState,
	setState,
	getMovieById,
	updateMovie,
	resetGenres,
	refreshMovieCache
} from './database'

// Create and start queue
const q = queue({
	autostart: true,
	concurrency: config.MAX_CONNECTIONS,
	timeout: 5000,
	results: []
})

// On every job finish
q.on('success', () => {
	// Change loading bar when queue updates
	const {queueTotal} = getState()
	setState({loading: Math.round(q.length / queueTotal * 100)})
})

q.on('end', () => {
	broadcast('Services queue completed.')
	setState({loading: 0})

}
        
        // dl / complete
        
        if(tor[CONST.TORRENT_COMPLETE])
        {
            groups.cat["cat_com"] = 1;
        }
        else
        {
            groups.cat["cat_dls"] = 1;
        }
        
        // active / inactive
        
        if((tor[CONST.TORRENT_DOWNSPEED] > (this.settings["queue.slow_dl_threshold"] || 103)) ||
           (tor[CONST.TORRENT_UPSPEED] > (this.settings["queue.slow_ul_threshold"] || 103)))
        {
            groups.cat["cat_act"] = 1;
        }
        else
        {
            groups.cat["cat_iac"] = 1;
        }
        
        // update group counts
        // TODO: Move this elsewhere!
        (function(groups, oldGroups) {
            if (!oldGroups)
            {
                Object.each(groups.cat, function(_, cat)
                {

lbls.each(function(lbl) {
				groups.lbl["lbl_" + encodeID(lbl)] = 1;
			});
		}

		// Categories: Downloading/Completed
		if (tor[CONST.TORRENT_PROGRESS] < 1000) {
			groups.cat["cat_dls"] = 1;
		}
		else {
			groups.cat["cat_com"] = 1;
		}

		// Categories: Active/Inactive
		if (
			(tor[CONST.TORRENT_DOWNSPEED] > (this.settings["queue.slow_dl_threshold"] || 103)) ||
			(tor[CONST.TORRENT_UPSPEED] > (this.settings["queue.slow_ul_threshold"] || 103))
		) {
			groups.cat["cat_act"] = 1;
		}
		else {
			groups.cat["cat_iac"] = 1;
		}

		// Update group counts
		// TODO: Move this elsewhere!
		(function(groups, oldGroups) {
			if (!oldGroups) {
				Object.each(groups.cat, function(_, cat) {
					++this.categories[cat];
				}, this);
			}

groups.lbl["lbl_" + encodeID(lbl)] = 1;
			});
		}

		// Categories: Downloading/Completed
		if (tor[CONST.TORRENT_PROGRESS] < 1000) {
			groups.cat["cat_dls"] = 1;
		}
		else {
			groups.cat["cat_com"] = 1;
		}

		// Categories: Active/Inactive
		if (
			(tor[CONST.TORRENT_DOWNSPEED] > (this.settings["queue.slow_dl_threshold"] || 103)) ||
			(tor[CONST.TORRENT_UPSPEED] > (this.settings["queue.slow_ul_threshold"] || 103))
		) {
			groups.cat["cat_act"] = 1;
		}
		else {
			groups.cat["cat_iac"] = 1;
		}

		// Update group counts
		// TODO: Move this elsewhere!
		(function(groups, oldGroups) {
			if (!oldGroups) {
				Object.each(groups.cat, function(_, cat) {
					++this.categories[cat];
				}, this);
			}
			else {