Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
_onOffsetCommit(err, topicPartitions) {
if (err) {
// NO_OFFSET is a "soft error" meaning that the same
// offset is already committed, which occurs because of
// auto-commit (e.g. if nothing was done by the producer
// on this partition since last commit).
if (err === kafka.CODES.ERRORS.ERR__NO_OFFSET) {
return undefined;
}
this._log.error('error committing offset to kafka',
{ errorCode: err });
return undefined;
}
this._log.debug('commit offsets callback',
{ topicPartitions });
return undefined;
}
import * as Kafka from 'node-rdkafka'
import * as winston from 'winston'
import { ConnectionDeadError } from './errors'
import { Options } from './types'
const ErrorCode = Kafka.CODES.ERRORS
const FLUSH_TIMEOUT = 10000 // ms
export abstract class KafkaBasicProducer {
public client: Kafka.Producer
protected logger: winston.Logger
protected debug: boolean
protected dying: boolean
protected dead: boolean
protected flushing: boolean
constructor(conf: any, topicConf: any = {}, options: Options = {}) {
this.dying = false
this.dead = false
this.flushing = false
this.client = new Kafka.Producer(conf, topicConf)
import * as Kafka from 'node-rdkafka'
import * as _ from 'lodash'
import * as bluebird from 'bluebird'
import * as winston from 'winston'
import { KafkaMessage, TopicPartition, Options } from './types'
import { ConnectionDeadError } from './errors'
const DEFAULT_CONSUME_SIZE = 100
const DEFAULT_CONCURRENT = 100
const DEFAULT_AUTO_COMMIT_INTERVAL = 1000 // ms
const ErrorCode = Kafka.CODES.ERRORS
export abstract class KafkaBasicConsumer {
public consumer: Kafka.KafkaConsumer
protected logger: winston.Logger
protected debug: boolean
protected dying: boolean
protected dead: boolean
protected topics: string[]
constructor(conf: any, topicConf: any = {}, options: Options = {}) {
this.dying = false
this.dead = false
this.topics = []
conf['auto.commit.interval.ms'] =
conf['auto.commit.interval.ms'] || DEFAULT_AUTO_COMMIT_INTERVAL
import * as Kafka from 'node-rdkafka'
export const KfkNativeErrorCode = Kafka.CODES.ERRORS
export const KfkErrorCode = {
UNDEFINED: -1,
CONNECTION_DEAD: 1,
}
export class KfkError extends Error {
public code: number = KfkErrorCode.UNDEFINED
}
export class ConnectionDeadError extends KfkError {
constructor(message: string) {
super(message)
this.code = KfkErrorCode.CONNECTION_DEAD
}
}
this._shouldFailWhenSchemaIsMissing = opts.shouldFailWhenSchemaIsMissing === true;
/** @type {Array.} Instanciated producers. */
this._producers = [];
/** @type {Array.} Instantiated consumers. */
this._consumers = [];
/** @type {Array.} Instantiated consumers. */
this._consumersStream = [];
});
/**
* Expose the node-rdkafka library's CODES constants.
*
* @type {Object}
*/
KafkaAvro.CODES = Kafka.CODES;
KafkaAvro.mixin(Producer);
KafkaAvro.mixin(Consumer);
/**
* Get the bunyan logger.
*
* @return {bunyan.Logger} The bunyan logger, singleton.
* @static
*/
KafkaAvro.getLogger = function () {
return rootLog;
};
/**
* Initialize the library, fetch schemas and register them locally.