Skip to content

Commit

Permalink
Merge pull request #16108 from carlosmiei/fast-build
Browse files Browse the repository at this point in the history 
Fast transpiling using multi-processing
  • Loading branch information
@kroitor
kroitor committed Dec 16, 2022
2 parents ab45aa5 + d06adcf commit 0ba94a8
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 18 deletions.
35 changes: 27 additions & 8 deletions build/transpile.js
View file
Expand Up @@ -5,6 +5,7 @@
"use strict";

const fs = require ('fs')
, { promisify } = require("util")
, log = require ('ololog').unlimited
, _ = require ('ansicolor').nice
, errors = require ('../js/base/errors.js')
Expand All @@ -26,8 +27,8 @@ const fs = require ('fs')
, Exchange = require ('.' + baseExchangeJsFile)
, tsFilename = './ccxt.d.ts'
, pythonCodingUtf8 = '# -*- coding: utf-8 -*-'
const {ids: exchanges} = require("../exchanges.json");

const exchangeIds = require("../exchanges.json").ids;
class Transpiler {

getCommonRegexes () {
Expand Down Expand Up @@ -1446,6 +1447,20 @@ class Transpiler {

// ========================================================================

async getTSClassDeclarationsAllFiles (ids, folder, extension = '.js')  {
const files = fs.readdirSync (folder).filter (file => ids.includes (basename (file, extension)))
const promiseReadFile = promisify (fs.readFile);
const fileArray = await Promise.all (files.map (file => promiseReadFile (folder + file, 'utf8')));
const classComponents = await Promise.all (fileArray.map (file => this.getClassDeclarationMatches (file)));

const classes = {}
classComponents.forEach ( elem => classes[elem[1]] = elem[2] );

return classes
}

// ========================================================================

exportTypeScriptClassNames (file, classes) {

log.bright.cyan ('Exporting TypeScript class names →', file.yellow)
Expand All @@ -1471,7 +1486,9 @@ class Transpiler {
replaceInFile (file, regex, replacement)
}

exportTypeScriptDeclarations (file, classes) {
async exportTypeScriptDeclarations (file, jsFolder) {

const classes = await this.getTSClassDeclarationsAllFiles (exchangeIds, jsFolder);

this.exportTypeScriptClassNames (file, classes)
this.exportTypeScriptExchangeIds (file, classes)
Expand Down Expand Up @@ -1912,14 +1929,15 @@ class Transpiler {

// ============================================================================

transpileEverything (force = false, child = false) {
async transpileEverything (force = false, child = false) {

// default pattern is '.js'
const exchanges = process.argv.slice (2).filter (x => !x.startsWith ('--'))
, python2Folder = './python/ccxt/'
, python3Folder = './python/ccxt/async_support/'
, phpFolder = './php/'
, phpAsyncFolder = './php/async/'
, jsFolder = './js/'
, options = { python2Folder, python3Folder, phpFolder, phpAsyncFolder, exchanges }

if (!child) {
Expand All @@ -1931,7 +1949,7 @@ class Transpiler {

//*

const classes = this.transpileDerivedExchangeFiles ('./js/', options, '.js', force, child || exchanges.length)
const classes = this.transpileDerivedExchangeFiles (jsFolder, options, '.js', force, child || exchanges.length)

if (classes === null) {
log.bright.yellow ('0 files transpiled.')
Expand All @@ -1944,7 +1962,7 @@ class Transpiler {
this.transpileBaseMethods ()
// HINT: if we're going to support specific class definitions
// this process won't work anymore as it will override the definitions
this.exportTypeScriptDeclarations (tsFilename, classes)
await this.exportTypeScriptDeclarations (tsFilename, jsFolder)

//*/

Expand Down Expand Up @@ -1994,10 +2012,11 @@ if (require.main === module) { // called directly like `node module`
} else if (errors) {
transpiler.transpileErrorHierarchy ({ tsFilename })
} else if (multiprocess) {
const exchanges = require ('../exchanges.json').ids
parallelizeTranspiling (exchanges)
parallelizeTranspiling (exchangeIds)
} else {
transpiler.transpileEverything (force, child)
(async () => {
await transpiler.transpileEverything (force, child)
})()
}

} else { // if required as a module
Expand Down
19 changes: 12 additions & 7 deletions build/transpileWS.js
View file
Expand Up @@ -18,6 +18,8 @@ const fs = require ('fs')
, Exchange = require ('../js/pro/base/Exchange.js')
, tsFilename = './ccxt.d.ts'

const wsExchangeIds = require ('../exchanges.json').ws

// ============================================================================

class CCXTProTranspiler extends Transpiler {
Expand Down Expand Up @@ -242,28 +244,30 @@ class CCXTProTranspiler extends Transpiler {

// -----------------------------------------------------------------------

exportTypeScriptDeclarations (file, classes) {
async exportTypeScriptDeclarations (file, jsFolder) {

const classes = await this.getTSClassDeclarationsAllFiles (wsExchangeIds, jsFolder);
this.exportTypeScriptClassNames (file, classes)
}

// -----------------------------------------------------------------------

transpileEverything (force = false, child = false) {
async transpileEverything (force = false, child = false) {

// default pattern is '.js'
// const [ /* node */, /* script */, pattern ] = process.argv.filter (x => !x.startsWith ('--'))
const exchanges = process.argv.slice (2).filter (x => !x.startsWith ('--'))
// , python2Folder = './python/ccxtpro/', // CCXT Pro does not support Python 2
, python3Folder = './python/ccxt/pro/'
, phpAsyncFolder = './php/pro/'
, jsFolder = './js/pro/'
, options = { /* python2Folder, */ python3Folder, phpAsyncFolder, exchanges }

// createFolderRecursively (python2Folder)
createFolderRecursively (python3Folder)
createFolderRecursively (phpAsyncFolder)

const classes = this.transpileDerivedExchangeFiles ('./js/pro/', options, '.js', force, child || exchanges.length)
const classes = this.transpileDerivedExchangeFiles (jsFolder, options, '.js', force, child || exchanges.length)

if (child) {
return
Expand All @@ -280,7 +284,7 @@ class CCXTProTranspiler extends Transpiler {

// HINT: if we're going to support specific class definitions
// this process won't work anymore as it will override the definitions
this.exportTypeScriptDeclarations (tsFilename, classes)
await this.exportTypeScriptDeclarations (tsFilename, jsFolder)

//*/

Expand Down Expand Up @@ -309,10 +313,11 @@ if (require.main === module) {
log.bright.green ({ force })
}
if (multiprocess) {
const exchanges = require ('../exchanges.json').ws
parallelizeTranspiling (exchanges)
parallelizeTranspiling (wsExchangeIds)
} else {
transpiler.transpileEverything (force)
(async () => {
await transpiler.transpileEverything (force, child)
})()
}

} else {
Expand Down
8 changes: 5 additions & 3 deletions package.json
View file
Expand Up @@ -18,7 +18,8 @@
"scripts": {
"docker": "docker-compose run --rm ccxt",
"build": "npm run pre-transpile && npm run transpile && npm run post-transpile && npm run update-badges",
"force-build": "npm run pre-transpile && npm run force-transpile && npm run post-transpile && npm run update-badges",
"force-build": "npm run pre-transpile && npm run force-transpile-fast && npm run post-transpile && npm run update-badges",
"force-build-slow": "npm run pre-transpile && npm run force-transpile && npm run post-transpile && npm run update-badges",
"pre-transpile": "npm run export-exchanges && npm run vss && npm run copy-python-files && npm run check-js-syntax && npm run browserify",
"post-transpile": "npm run check-python-syntax && npm run check-php-syntax",
"test-ws": "npm run build && node run-tests-ws",
Expand Down Expand Up @@ -53,6 +54,7 @@
"transpileRest": "node build/transpile",
"transpileWs": "node build/transpileWS",
"force-transpile": "npm run force-transpileRest && npm run force-transpileWs",
"force-transpile-fast": "npm run dev-force-transpile",
"dev-force-transpile": "npm run fast-force-transpileRest && npm run fast-force-transpileWs",
"force-transpileRest": "node build/transpile --force",
"fast-force-transpileRest": "node build/transpile.js --multiprocess",
Expand Down Expand Up @@ -91,7 +93,8 @@
"eslint-config-airbnb-base": "15.0.0",
"eslint-plugin-import": "2.25.4",
"https-proxy-agent": "^5.0.1",
"ololog": "1.1.155"
"ololog": "1.1.155",
"replace-in-file": "^6.3.5"
},
"author": {
"name": "Igor Kroitor",
Expand Down Expand Up @@ -625,7 +628,6 @@
},
"ethereum": "0x26a3CB49578F07000575405a57888681249c35Fd",
"dependencies": {
"replace-in-file": "^6.3.5",
"ws": "^8.8.1"
}
}

0 comments on commit 0ba94a8

Please sign in to comment.