Skip to content
This repository has been archived by the owner on Jan 13, 2024. It is now read-only.

Commit

Permalink
Add support for symlink and pnpm. (#1060)
Browse files Browse the repository at this point in the history
  • Loading branch information
@erossignon
erossignon committed Mar 31, 2021
1 parent 79cca54 commit eb8921a
Show file tree
Hide file tree
Showing 23 changed files with 688 additions and 183 deletions.
23 changes: 23 additions & 0 deletions README.md
View file
Expand Up @@ -311,3 +311,26 @@ You could check on **Unix systems** (Linux/macOS) in `bash`:
```bash
$ printenv | grep NODE
```

## Advanced

### exploring virtual file system embedded in debug mode

When you are using the `--debug` flag when building your executable,
`pkg` add the ability to display the content of the virtual file system
and the symlink table on the console, when the application starts,
providing that the environement variable DEBUG_PKG is set.
This feature can be useful to inspect if symlinks are correctly handled,
and check that all the required files for your application are properly
incorporated to the final executable.

$ pkg --debug app.js -o output
$ DEBUG_PKG output

or

C:\> pkg --debug app.js -o output.exe
C:\> set DEBUG_PKG=1
C:\> output.exe

Note: make sure not to use --debug flag in production.
8 changes: 8 additions & 0 deletions lib/follow.js
View file
Expand Up @@ -2,6 +2,7 @@ import { core, sync } from 'resolve';
import assert from 'assert';
import fs from 'fs';
import path from 'path';
import { toNormalizedRealPath } from '../prelude/common';

Object.keys(core).forEach((key) => {
// 'resolve' hardcodes the list to host's one, but i need
Expand Down Expand Up @@ -78,6 +79,13 @@ export function follow(x, opts) {
if (opts.packageFilter) opts.packageFilter(config, base);
return config;
},

/** function to synchronously resolve a potential symlink to its real path */
// realpathSync?: (file: string) => string;
realpathSync: (file) => {
const file2 = toNormalizedRealPath(file);
return file2;
},
})
);
});
Expand Down
13 changes: 8 additions & 5 deletions lib/index.js
View file
Expand Up @@ -313,8 +313,7 @@ export async function exec(argv2) {
if (!outputPath) {
if (inputJson && inputJson.pkg) {
outputPath = inputJson.pkg.outputPath;
} else
if (configJson && configJson.pkg) {
} else if (configJson && configJson.pkg) {
outputPath = configJson.pkg.outputPath;
}
outputPath = outputPath || '';
Expand Down Expand Up @@ -467,17 +466,21 @@ export async function exec(argv2) {

let records;
let entrypoint = inputFin;
let symLinks;
const addition = isConfiguration(input) ? input : undefined;

const walkResult = await walk(marker, entrypoint, addition, params);
entrypoint = walkResult.entrypoint;

records = walkResult.records;
symLinks = walkResult.symLinks;

const refineResult = refine(records, entrypoint);
const refineResult = refine(records, entrypoint, symLinks);
entrypoint = refineResult.entrypoint;
records = refineResult.records;
symLinks = refineResult.symLinks;

const backpack = packer({ records, entrypoint, bytecode });
const backpack = packer({ records, entrypoint, bytecode, symLinks });

log.debug('Targets:', JSON.stringify(targets, null, 2));

Expand All @@ -495,7 +498,7 @@ export async function exec(argv2) {
}

const slash = target.platform === 'win' ? '\\' : '/';
await producer({ backpack, bakes, slash, target });
await producer({ backpack, bakes, slash, target, symLinks });
if (target.platform !== 'win') {
await plusx(target.output);
}
Expand Down
90 changes: 41 additions & 49 deletions lib/packer.js
View file
Expand Up @@ -24,6 +24,11 @@ const commonText = fs.readFileSync(
'utf8'
);

const diagnosticText = fs.readFileSync(
require.resolve('../prelude/diagnostic.js'),
'utf8'
);

function itemsToText(items) {
const len = items.length;
return len.toString() + (len % 10 === 1 ? ' item' : ' items');
Expand All @@ -48,12 +53,14 @@ export default function packer({ records, entrypoint, bytecode }) {
assert(record[STORE_STAT], 'packer: no STORE_STAT');

assert(
record[STORE_BLOB] || record[STORE_CONTENT] || record[STORE_LINKS]
record[STORE_BLOB] ||
record[STORE_CONTENT] ||
record[STORE_LINKS] ||
record[STORE_STAT]
);

if (record[STORE_BLOB] && !bytecode) {
delete record[STORE_BLOB];

if (!record[STORE_CONTENT]) {
// TODO make a test for it?
throw wasReported(
Expand All @@ -66,7 +73,6 @@ export default function packer({ records, entrypoint, bytecode }) {
);
}
}

for (const store of [
STORE_BLOB,
STORE_CONTENT,
Expand All @@ -88,68 +94,54 @@ export default function packer({ records, entrypoint, bytecode }) {
}
} else if (store === STORE_LINKS) {
if (Array.isArray(value)) {
const buffer = Buffer.from(JSON.stringify(value));
const dedupedValue = [...new Set(value)];
log.debug('files & folders deduped = ', dedupedValue);
const buffer = Buffer.from(JSON.stringify(dedupedValue));
stripes.push({ snap, store, buffer });
} else {
assert(false, 'packer: bad STORE_LINKS');
}
} else if (store === STORE_STAT) {
if (typeof value === 'object') {
// reproducible
delete value.atime;
delete value.atimeMs;
delete value.mtime;
delete value.mtimeMs;
delete value.ctime;
delete value.ctimeMs;
delete value.birthtime;
delete value.birthtimeMs;
// non-date
delete value.blksize;
delete value.blocks;
delete value.dev;
delete value.gid;
delete value.ino;
delete value.nlink;
delete value.rdev;
delete value.uid;
if (!value.isFile()) value.size = 0;
// portable
const newStat = { ...value };
newStat.isFileValue = value.isFile();
newStat.isDirectoryValue = value.isDirectory();
newStat.isSocketValue = value.isSocket();
const buffer = Buffer.from(JSON.stringify(newStat));
stripes.push({ snap, store, buffer });
} else {
assert(false, 'packer: bad STORE_STAT');
assert(false, 'packer: unknown store');
}
} else {
assert(false, 'packer: unknown store');
}
}

if (record[STORE_CONTENT]) {
const disclosed = isDotJS(file) || isDotJSON(file);
log.debug(
disclosed
? 'The file was included as DISCLOSED code (with sources)'
: 'The file was included as asset content',
file
);
} else if (record[STORE_BLOB]) {
log.debug('The file was included as bytecode (no sources)', file);
} else if (record[STORE_LINKS]) {
const value = record[STORE_LINKS];
log.debug(
`The directory files list was included (${itemsToText(value)})`,
file
);
if (record[STORE_CONTENT]) {
const disclosed = isDotJS(file) || isDotJSON(file);
log.debug(
disclosed
? 'The file was included as DISCLOSED code (with sources)'
: 'The file was included as asset content',
file
);
} else if (record[STORE_BLOB]) {
log.debug('The file was included as bytecode (no sources)', file);
} else if (record[STORE_LINKS]) {
const link = record[STORE_LINKS];
log.debug(
`The directory files list was included (${itemsToText(link)})`,
file
);
}
}
}
}

const prelude = `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT) { ${bootstrapText}\n})(function (exports) {\n${commonText}\n},\n${'%VIRTUAL_FILESYSTEM%'}\n,\n${'%DEFAULT_ENTRYPOINT%'}\n);`;
const prelude =
`return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) {
${bootstrapText}${
log.debugMode ? diagnosticText : ''
}\n})(function (exports) {\n${commonText}\n},\n` +
`%VIRTUAL_FILESYSTEM%` +
`\n,\n` +
`%DEFAULT_ENTRYPOINT%` +
`\n,\n` +
`%SYMLINKS%` +
`\n);`;

return { prelude, entrypoint, stripes };
}
22 changes: 16 additions & 6 deletions lib/producer.js
View file
Expand Up @@ -166,7 +166,7 @@ function nativePrebuildInstall(target, nodeFile) {
return nativeFile;
}

export default function producer({ backpack, bakes, slash, target }) {
export default function producer({ backpack, bakes, slash, target, symLinks }) {
return new Promise((resolve, reject) => {
if (!Buffer.alloc) {
throw wasReported(
Expand All @@ -186,6 +186,12 @@ export default function producer({ backpack, bakes, slash, target }) {
if (!vfs[snap]) vfs[snap] = {};
}

const snapshotSymLinks = {};
for (const [key, value] of Object.entries(symLinks)) {
const k = snapshotify(key, slash);
const v = snapshotify(value, slash);
snapshotSymLinks[k] = v;
}
let meter;
let count = 0;

Expand Down Expand Up @@ -300,12 +306,16 @@ export default function producer({ backpack, bakes, slash, target }) {
makePreludeBufferFromPrelude(
replaceDollarWise(
replaceDollarWise(
prelude,
'%VIRTUAL_FILESYSTEM%',
JSON.stringify(vfs)
replaceDollarWise(
prelude,
'%VIRTUAL_FILESYSTEM%',
JSON.stringify(vfs)
),
'%DEFAULT_ENTRYPOINT%',
JSON.stringify(entrypoint)
),
'%DEFAULT_ENTRYPOINT%',
JSON.stringify(entrypoint)
'%SYMLINKS%',
JSON.stringify(snapshotSymLinks)
)
)
)
Expand Down
43 changes: 27 additions & 16 deletions lib/refiner.js
View file
@@ -1,9 +1,11 @@
import path from 'path';
import chalk from 'chalk';
import {
STORE_LINKS,
retrieveDenominator,
substituteDenominator,
} from '../prelude/common';
import { log } from './log';

const win32 = process.platform === 'win32';

Expand All @@ -22,7 +24,6 @@ function purgeTopDirectories(records) {
if (records[file]) {
const record = records[file];
const links = record[STORE_LINKS];

if (links && links.length === 1) {
if (!hasParent(file, records)) {
const file2 = path.join(file, links[0]);
Expand All @@ -33,8 +34,8 @@ function purgeTopDirectories(records) {
const record3 = records[file3];
const links3 = record3[STORE_LINKS];
if (links3) {
// eslint-disable-next-line no-param-reassign
delete records[file];
log.debug(chalk.cyan('Deleting record file :', file));
found = true;
}
}
Expand All @@ -47,31 +48,41 @@ function purgeTopDirectories(records) {
}
}

function denominate(records, entrypoint, denominator) {
function denominate(records, entrypoint, denominator, symLinks) {
const newRecords = {};

for (const file in records) {
if (records[file]) {
let snap = substituteDenominator(file, denominator);
const makeSnap = (file) => {
let snap = substituteDenominator(file, denominator);

if (win32) {
if (snap.slice(1) === ':') snap += '\\';
} else if (snap === '') {
snap = '/';
}

newRecords[snap] = records[file];
if (win32) {
if (snap.slice(1) === ':') snap += '\\';
} else if (snap === '') {
snap = '/';
}
}

return snap;
};
// eslint-disable-next-line guard-for-in
for (const file in records) {
const snap = makeSnap(file);
newRecords[snap] = records[file];
}
const tmpSymLinks = symLinks;
symLinks = {};
for (const [key, value] of Object.entries(tmpSymLinks)) {
const key1 = makeSnap(key);
const value1 = makeSnap(value);
symLinks[key1] = value1;
}
return {
records: newRecords,
entrypoint: substituteDenominator(entrypoint, denominator),
symLinks,
};
}

export default function refiner(records, entrypoint) {
export default function refiner(records, entrypoint, symLinks) {
purgeTopDirectories(records);
const denominator = retrieveDenominator(Object.keys(records));
return denominate(records, entrypoint, denominator);
return denominate(records, entrypoint, denominator, symLinks);
}

0 comments on commit eb8921a

Please sign in to comment.