Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
}
this._state = 'loading';
let { id, utterance, preprocessed, target_code: oldTargetCode } = line;
this._utterance = utterance;
if (!oldTargetCode)
oldTargetCode = preprocessed;
if (!id)
id = this._serial;
const parsed = await this._parser.sendUtterance(utterance, /* tokenized */ false, /* context */ undefined);
if (oldTargetCode) {
try {
const program = ThingTalk.NNSyntax.fromNN(oldTargetCode.split(' '), parsed.entities);
await program.typecheck(this._schemas);
} catch(e) {
console.log(`Sentence ${id}'s existing code is incorrect: ${e}`); //'
oldTargetCode = undefined;
}
}
if (oldTargetCode)
parsed.candidates.unshift({ code: oldTargetCode.split(' '), score: 'Infinity' });
this._state = 'top3';
this._id = id;
this._preprocessed = parsed.tokens.join(' ');
this._entities = parsed.entities;
this._candidates = (await Promise.all(parsed.candidates.map(async (cand) => {
try {
const program = ThingTalk.NNSyntax.fromNN(cand.code, parsed.entities);
//
// This file is part of Genie
//
// Copyright 2018-2019 The Board of Trustees of the Leland Stanford Junior University
//
// Author: Silei Xu
// Giovanni Campagna
//
// See COPYING for details
"use strict";
const Stream = require('stream');
const ThingTalk = require('thingtalk');
const Grammar = ThingTalk.Grammar;
const NNSyntax = ThingTalk.NNSyntax;
const i18n = require('./i18n');
const Utils = require('./utils');
class ParaphraseValidator {
constructor(schemaRetriever, tokenizer, locale, row, counter = {}) {
this._schemas = schemaRetriever;
this._tokenizer = tokenizer;
this._locale = locale;
this._noIdea = i18n.get(locale).NO_IDEA;
this._counter = counter;
this.id = row.id;
this.target_code = row.target_code;
this.paraphrase = row.paraphrase;
this.synthetic = row.synthetic;
TokenizerService.tokenize(language, utterance)]).then(([program, { tokens: preprocessed, entities }]) => {
let target_code = ThingTalk.NNSyntax.toNN(program, entities);
for (let name in entities) {
if (name === '$used') continue;
throw new BadRequestError('Unused entity ' + name);
}
return example.create(dbClient, {
utterance: utterance,
preprocessed: preprocessed.join(' '),
target_code: target_code.join(' '),
target_json: '', // FIXME
type: 'turking' + batchId,
language: language,
is_base: 0
});
});
}
// Author: Giovanni Campagna
//
// See COPYING for details
"use strict";
const assert = require('assert');
const stream = require('stream');
const path = require('path');
const events = require('events');
const ThingTalk = require('thingtalk');
const Ast = ThingTalk.Ast;
const Type = ThingTalk.Type;
const Grammar = ThingTalk.Grammar;
const SchemaRetriever = ThingTalk.SchemaRetriever;
const NNSyntax = ThingTalk.NNSyntax;
const Units = ThingTalk.Units;
const { clean, makeDummyEntities } = require('../utils');
const $runtime = require('./runtime');
const importGenie = require('../genie-compiler');
const { typeToStringSafe } = require('../../languages/ast_manip');
const i18n = require('../i18n');
function identity(x) {
return x;
}
const BASIC_TARGET_GEN_SIZE = 100000;
const CONTEXTUAL_TARGET_GEN_SIZE = 10000;
let type = ex.args[name];
// avoid examples such as "post __" for both text and picture (should be "post picture" without slot for picture)
if (type.isEntity && type.type === 'tt:picture')
return null;
}
// turn the declaration into a program
let newprogram = ex.toProgram();
let slots = [];
let slotTypes = {};
for (let name in ex.args) {
slotTypes[name] = String(ex.args[name]);
slots.push(name);
}
let code = ThingTalk.NNSyntax.toNN(newprogram, {});
let monitorable;
if (ex.type === 'stream')
monitorable = true;
else if (ex.type === 'action')
monitorable = false;
else if (ex.type === 'query')
monitorable = ex.value.schema.is_monitorable;
else
monitorable = false;
return { utterance: ex.utterances[0],
type: ex.type,
monitorable: monitorable,
target: {
example_id: ex.id, code: code, entities: {}, slotTypes: slotTypes, slots: slots } };
},
const contexts = await Promise.all(minibatch.map(async (contextCode) => {
const code = contextCode.split(' ');
const entities = makeDummyEntities(contextCode);
const program = ThingTalk.NNSyntax.fromNN(code, entities);
await program.typecheck(this._generator.schemas, false);
return new $runtime.Context(code, program, entities);
}));
_toNN(program) {
let clone = {};
Object.assign(clone, this._entities);
return ThingTalk.NNSyntax.toNN(program, this._tokens, clone);
}
Intent.parse = async function parse(json, schemaRetriever, context) {
if ('program' in json)
return Intent.fromThingTalk(await ThingTalk.Grammar.parseAndTypecheck(json.program, schemaRetriever, true), context);
let { code, entities } = json;
for (let name in entities) {
if (name.startsWith('SLOT_')) {
let slotname = json.slots[parseInt(name.substring('SLOT_'.length))];
let slotType = ThingTalk.Type.fromString(json.slotTypes[slotname]);
let value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);
entities[name] = value;
}
}
const thingtalk = ThingTalk.NNSyntax.fromNN(code, entities);
await thingtalk.typecheck(schemaRetriever, true);
return Intent.fromThingTalk(thingtalk, context);
};
entities = tokenized.entities;
}
const targetNN = ThingTalk.NNSyntax.toNN(targetCommand, tokens, entities);
this.push({
id: 'dlg' + dialog.id + ':' + i,
context: contextNN.join(' '),
preprocessed: tokens.join(' '),
target_code: targetNN.join(' ')
});
context = this._applyReplyToContext(context, targetCommand);
contextEntities = {};
if (context !== null)
contextNN = ThingTalk.NNSyntax.toNN(context, '', contextEntities, { allocateEntities: true });
else
contextNN = ['null'];
}
}
Intent.parse = async function parse(json, schemaRetriever, context) {
if ('program' in json)
return Intent.fromThingTalk(await ThingTalk.Grammar.parseAndTypecheck(json.program, schemaRetriever, true), context);
let { code, entities } = json;
for (let name in entities) {
if (name.startsWith('SLOT_')) {
let slotname = json.slots[parseInt(name.substring('SLOT_'.length))];
let slotType = ThingTalk.Type.fromString(json.slotTypes[slotname]);
let value = ThingTalk.Ast.Value.fromJSON(slotType, entities[name]);
entities[name] = value;
}
}
const thingtalk = ThingTalk.NNSyntax.fromNN(code, entities);
await thingtalk.typecheck(schemaRetriever, true);
return Intent.fromThingTalk(thingtalk, context);
};