Skip to content

Commit

Permalink
address comments + fix bug
Browse files Browse the repository at this point in the history
  • Loading branch information
@usmanmani1122
usmanmani1122 committed Jan 30, 2025
1 parent 5d54940 commit ef3d10b
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 68 deletions.
4 changes: 3 additions & 1 deletion packages/synthetic-chain/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,15 @@
"chalk": "^5.4.1",
"cosmjs-types": "^0.9.0",
"execa": "^9.5.2",
"glob": "^11.0.0"
"glob": "^11.0.0",
"tmp": "0.2.3"
},
"devDependencies": {
"@agoric/cosmic-proto": "0.5.0-u18.5",
"@types/better-sqlite3": "^7.6.11",
"@types/glob": "^8.1.0",
"@types/node": "^18.19.50",
"@types/tmp": "0.2.6",
"ava": "^6.2.0",
"ts-blank-space": "^0.5.0",
"tsup": "^8.3.5",
Expand Down
2 changes: 1 addition & 1 deletion packages/synthetic-chain/src/cli/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ switch (cmd) {
console.log(chalk.cyan.bold(`Testing ${proposal.proposalName}`));
const image = imageNameForProposal(proposal, 'test');
bakeTarget(image.target, values.dry);
runTestImage(proposal);
runTestImage({ proposal });
// delete the image to reclaim disk space. The next build
// will use the build cache.
execSync('docker system df', { stdio: 'inherit' });
Expand Down
85 changes: 74 additions & 11 deletions packages/synthetic-chain/src/cli/dockerfileGen.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,12 @@ ENV UPGRADE_TO=${to}
RUN echo '. /usr/src/upgrade-test-scripts/env_setup.sh' >> ~/.bashrc
# copy scripts
COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/run_prepare_zero.sh /usr/src/upgrade-test-scripts/
${createCopyCommand(
[],
'./upgrade-test-scripts/env_setup.sh',
'./upgrade-test-scripts/run_prepare_zero.sh',
'/usr/src/upgrade-test-scripts/',
)}
SHELL ["/bin/bash", "-c"]
# this is the only layer that starts ag0
RUN /usr/src/upgrade-test-scripts/run_prepare_zero.sh
Expand Down Expand Up @@ -75,8 +80,18 @@ ENV \
UPGRADE_INFO=${JSON.stringify(encodeUpgradeInfo(upgradeInfo))} \
SKIP_PROPOSAL_VALIDATION=${skipProposalValidation}
COPY --exclude=host --exclude=test --exclude=test.sh --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/run_prepare.sh ./upgrade-test-scripts/start_to_to.sh /usr/src/upgrade-test-scripts/
${createCopyCommand(
['host', 'node_modules', 'test', 'test.sh'],
`./proposals/${path}`,
`/usr/src/proposals/${path}`,
)}
${createCopyCommand(
[],
'./upgrade-test-scripts/env_setup.sh',
'./upgrade-test-scripts/run_prepare.sh',
'./upgrade-test-scripts/start_to_to.sh',
'/usr/src/upgrade-test-scripts/',
)}
WORKDIR /usr/src/upgrade-test-scripts
SHELL ["/bin/bash", "-c"]
RUN ./run_prepare.sh ${path}
Expand All @@ -100,8 +115,19 @@ FROM ghcr.io/agoric/agoric-sdk:${sdkImageTag} as execute-${proposalName}
WORKDIR /usr/src/upgrade-test-scripts
# base is a fresh sdk image so set up the proposal and its dependencies
COPY --exclude=host --exclude=test --exclude=test.sh --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/run_execute.sh ./upgrade-test-scripts/start_to_to.sh ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
${createCopyCommand(
['host', 'node_modules', 'test', 'test.sh'],
`./proposals/${path}`,
`/usr/src/proposals/${path}`,
)}
${createCopyCommand(
['test.sh'],
'./upgrade-test-scripts/env_setup.sh',
'./upgrade-test-scripts/run_execute.sh',
'./upgrade-test-scripts/start_to_to.sh',
'./upgrade-test-scripts/install_deps.sh',
'/usr/src/upgrade-test-scripts/',
)}
RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${path}
COPY --link --from=prepare-${proposalName} /root/.agoric /root/.agoric
Expand All @@ -122,15 +148,27 @@ RUN ./run_execute.sh ${planName}
# EVAL ${proposalName}
FROM use-${lastProposal.proposalName} as eval-${proposalName}
COPY --exclude=host --exclude=test --exclude=test.sh --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
${createCopyCommand(
['host', 'node_modules', 'test', 'test.sh'],
`./proposals/${path}`,
`/usr/src/proposals/${path}`,
)}
WORKDIR /usr/src/upgrade-test-scripts
# First stage of this proposal so install its deps.
COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
${createCopyCommand(
[],
'./upgrade-test-scripts/install_deps.sh',
'/usr/src/upgrade-test-scripts/',
)}
RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${path}
COPY --link --chmod=755 ./upgrade-test-scripts/*eval* /usr/src/upgrade-test-scripts/
${createCopyCommand(
[],
'./upgrade-test-scripts/*eval*',
'/usr/src/upgrade-test-scripts/',
)}
SHELL ["/bin/bash", "-c"]
RUN ./run_eval.sh ${path}
`;
Expand All @@ -149,7 +187,12 @@ FROM ${previousStage}-${proposalName} as use-${proposalName}
WORKDIR /usr/src/upgrade-test-scripts
COPY --link --chmod=755 ./upgrade-test-scripts/run_use.sh ./upgrade-test-scripts/start_agd.sh /usr/src/upgrade-test-scripts/
${createCopyCommand(
[],
'./upgrade-test-scripts/run_use.sh',
'./upgrade-test-scripts/start_agd.sh',
'/usr/src/upgrade-test-scripts/',
)}
SHELL ["/bin/bash", "-c"]
RUN ./run_use.sh ${path}
ENTRYPOINT ./start_agd.sh
Expand All @@ -171,11 +214,19 @@ FROM use-${proposalName} as test-${proposalName}
# Previous stages copied excluding test files (see COPY above). It would be good
# to copy only missing files, but there may be none. Fortunately, copying extra
# does not invalidate other images because nothing depends on this layer.
COPY --exclude=host --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
${createCopyCommand(
['host', 'node_modules'],
`./proposals/${path}`,
`/usr/src/proposals/${path}`,
)}
WORKDIR /usr/src/upgrade-test-scripts
COPY --link --chmod=755 ./upgrade-test-scripts/run_test.sh /usr/src/upgrade-test-scripts/
${createCopyCommand(
[],
'./upgrade-test-scripts/run_test.sh',
'/usr/src/upgrade-test-scripts/',
)}
SHELL ["/bin/bash", "-c"]
ENTRYPOINT ./run_test.sh ${path}
`;
Expand All @@ -196,6 +247,18 @@ FROM ${useImage} as latest
},
};

export const createCopyCommand = (
exclusionList: Array<string>,
...files: Array<string>
) =>
[
'COPY',
'--link',
'--chmod=755',
...exclusionList.map(excluded => `--exclude=${excluded}`),
...files,
].join(' ');

export function writeBakefileProposals(
allProposals: ProposalInfo[],
platforms?: Platform[],
Expand Down
124 changes: 69 additions & 55 deletions packages/synthetic-chain/src/cli/run.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
import { spawnSync } from 'node:child_process';
import { existsSync, realpathSync } from 'node:fs';
import { resolve as resolvePath } from 'node:path';
import { basename, resolve as resolvePath } from 'node:path';
import { fileSync as createTempFile } from 'tmp';
import { ProposalInfo, imageNameForProposal } from './proposals.js';

const createMessageFile = () => {
const messageFileName = `${new Date().getTime()}.tmp`;
const messageFilePath = `/tmp/${messageFileName}`;
spawnSync('touch', [messageFilePath]);
return [messageFileName, messageFilePath];
};
const createMessageFile = (proposal: ProposalInfo) =>
createTempFile({ prefix: proposal.proposalName });

const executeHostScriptIfPresent = (
extraEnv: typeof process.env,
proposal: ProposalInfo,
scriptName: string,
) => {
Expand All @@ -19,7 +17,10 @@ const executeHostScriptIfPresent = (
console.log(
`Running script ${scriptName} for proposal ${proposal.proposalName}`,
);
spawnSync(scriptPath, { env: process.env, stdio: 'inherit' });
spawnSync(scriptPath, {
env: { ...process.env, ...extraEnv },
stdio: 'inherit',
});
}
};

Expand All @@ -45,77 +46,90 @@ const propagateSlogfile = env => {
];
};

export const runTestImage = (proposal: ProposalInfo) => {
const [messageFileName, messageFilePath] = createMessageFile();
export const runTestImage = ({
extraDockerArgs = [],
proposal,
removeContainerOnExit = true,
}: {
extraDockerArgs?: Array<string>;
proposal: ProposalInfo;
removeContainerOnExit?: boolean;
}) => {
const { name: messageFilePath, removeCallback: removeTempFileCallback } = createMessageFile(proposal);
const messageFileName = basename(messageFilePath);

const containerFilePath = `/root/${messageFileName}`;
process.env.MESSAGE_FILE_PATH = messageFilePath;

executeHostScriptIfPresent(proposal, 'before-test-run.sh');

console.log(`Running test image for proposal ${proposal.proposalName}`);
const { name } = imageNameForProposal(proposal, 'test');
spawnSync(
'docker',
[
'run',
'--env',
`MESSAGE_FILE_PATH=${containerFilePath}`,
'--mount',
`source=${messageFilePath},target=${containerFilePath},type=bind`,
'--network',
'host',
'--rm',
...propagateSlogfile(process.env),
name,
],
{ stdio: 'inherit' },
);
try {
executeHostScriptIfPresent(
{
MESSAGE_FILE_PATH: messageFilePath,
},
proposal,
'before-test-run.sh',
);

spawnSync('rm', ['--force', messageFilePath]);
console.log(`Running test image for proposal ${proposal.proposalName}`);
const { name } = imageNameForProposal(proposal, 'test');
spawnSync(
'docker',
[
'run',
'--env',
`MESSAGE_FILE_PATH=${containerFilePath}`,
'--mount',
`source=${messageFilePath},target=${containerFilePath},type=bind`,
'--network',
'host',
removeContainerOnExit && '--rm',
...propagateSlogfile(process.env),
...extraDockerArgs,
name,
]
.filter(Boolean)
.map(String),
{ stdio: 'inherit' },
);

executeHostScriptIfPresent(proposal, 'after-test-run.sh');
executeHostScriptIfPresent(
{
MESSAGE_FILE_PATH: messageFilePath,
},
proposal,
'after-test-run.sh',
);
} catch (err) {
removeTempFileCallback();
throw err;
}
};

export const debugTestImage = (proposal: ProposalInfo) => {
executeHostScriptIfPresent(proposal, 'before-test-run.sh');
const { name } = imageNameForProposal(proposal, 'test');
console.log(
`
Starting chain of test image for proposal ${proposal.proposalName}
To get an interactive shell in the container, use an IDE feature like "Attach Shell" or this command:'
docker exec -ti $(docker ps -q -f ancestor=${name}) bash
And within that shell:
cd /usr/src/proposals/${proposal.path} && ./test.sh
To edit files you can use terminal tools like vim, or mount the container in your IDE.
In VS Code the command is:
Dev Containers: Attach to Running Container...
`,
);

// start the chain with ports mapped
spawnSync(
'docker',
[
'run',
return runTestImage({
extraDockerArgs: [
'--entrypoint',
'/usr/src/upgrade-test-scripts/start_agd.sh',
'--interactive',
'--publish',
'1317:1317',
'--publish',
'9090:9090',
'--publish',
'26657:26657',
'--tty',
...propagateSlogfile(process.env),
name,
],
{ stdio: 'inherit' },
);
executeHostScriptIfPresent(proposal, 'after-test-run.sh');
proposal,
removeContainerOnExit: false,
});
};
16 changes: 16 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,14 @@ __metadata:
"@types/better-sqlite3": "npm:^7.6.11"
"@types/glob": "npm:^8.1.0"
"@types/node": "npm:^18.19.50"
"@types/tmp": "npm:0.2.6"
ava: "npm:^6.2.0"
better-sqlite3: "npm:^11.8.0"
chalk: "npm:^5.4.1"
cosmjs-types: "npm:^0.9.0"
execa: "npm:^9.5.2"
glob: "npm:^11.0.0"
tmp: "npm:0.2.3"
ts-blank-space: "npm:^0.5.0"
tsup: "npm:^8.3.5"
typescript: "npm:^5.7.3"
Expand Down Expand Up @@ -634,6 +636,13 @@ __metadata:
languageName: node
linkType: hard

"@types/tmp@npm:0.2.6":
version: 0.2.6
resolution: "@types/tmp@npm:0.2.6"
checksum: 10c0/a11bfa2cd8eaa6c5d62f62a3569192d7a2c28efdc5c17af0b0551db85816b2afc8156f3ca15ac76f0b142ae1403f04f44279871424233a1f3390b2e5fc828cd0
languageName: node
linkType: hard

"@vercel/nft@npm:^0.27.5":
version: 0.27.10
resolution: "@vercel/nft@npm:0.27.10"
Expand Down Expand Up @@ -3326,6 +3335,13 @@ __metadata:
languageName: node
linkType: hard

"tmp@npm:0.2.3":
version: 0.2.3
resolution: "tmp@npm:0.2.3"
checksum: 10c0/3e809d9c2f46817475b452725c2aaa5d11985cf18d32a7a970ff25b568438e2c076c2e8609224feef3b7923fa9749b74428e3e634f6b8e520c534eef2fd24125
languageName: node
linkType: hard

"to-regex-range@npm:^5.0.1":
version: 5.0.1
resolution: "to-regex-range@npm:5.0.1"
Expand Down

0 comments on commit ef3d10b

Please sign in to comment.