{"_id":"dagsby","_rev":"301014","name":"dagsby","description":"Gatsby library for orchestrating running data pipelines across workers","dist-tags":{"latest":"0.0.4"},"maintainers":[{"name":"kylemathews","email":""}],"time":{"modified":"2021-06-03T19:10:03.000Z","created":"2021-01-25T19:37:26.111Z","0.0.4":"2021-02-08T23:03:46.672Z","0.0.3":"2021-02-08T22:48:32.103Z","0.0.2":"2021-01-25T20:02:40.927Z","0.0.1":"2021-01-25T19:37:26.111Z"},"users":{},"author":{"name":"Kyle Mathews","email":"mathews.kyle@gmail.com"},"versions":{"0.0.4":{"name":"dagsby","version":"0.0.4","description":"Gatsby library for orchestrating running data pipelines across workers","main":"dist/index.js","scripts":{"build":"babel src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\"","prepare":"cross-env NODE_ENV=production npm run build","watch":"babel -w src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\""},"author":{"name":"Kyle Mathews","email":"mathews.kyle@gmail.com"},"license":"MIT","dependencies":{"@adobe/node-fetch-retry":"github:adobe/node-fetch-retry","@babel/cli":"^7.12.10","@babel/core":"^7.12.10","avsc":"^5.5.3","xxhashjs":"^0.2.2","babel-preset-gatsby-package":"^0.11.0-next.0","bottleneck":"^2.19.5","execa":"^5.0.0","fastq":"^1.10.0","fs-extra":"^9.0.1","lodash":"^4.17.20","p-queue":"^6.6.2","uuid":"^8.3.2","socket.io":"^2.0.0","socket.io-client":"^2.0.0","yargs":"^16.2.0"},"devDependencies":{"cross-env":"^7.0.3","detect-port":"^1.3.0","wait-on":"^5.2.1"},"_id":"dagsby@0.0.4","_nodeVersion":"14.15.4","_npmVersion":"6.14.10","dist":{"shasum":"f7d12bf4728728453e572fd2aa948746de1b25e8","size":401496,"noattachment":false,"key":"/dagsby/-/dagsby-0.0.4.tgz","tarball":"http://registry.cnpm.dingdandao.com/dagsby/download/dagsby-0.0.4.tgz"},"_npmUser":{"name":"kylemathews","email":"mathews.kyle@gmail.com"},"directories":{},"maintainers":[{"name":"kylemathews","email":""}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/dagsby_0.0.4_1612825426400_0.34055824817672886"},"_hasShrinkwrap":false,"publish_time":1612825426672,"_cnpm_publish_time":1612825426672},"0.0.3":{"name":"dagsby","version":"0.0.3","description":"Gatsby library for orchestrating running data pipelines across workers","main":"dist/index.js","scripts":{"build":"babel src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\"","prepare":"cross-env NODE_ENV=production npm run build","watch":"babel -w src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\""},"author":{"name":"Kyle Mathews","email":"mathews.kyle@gmail.com"},"license":"MIT","dependencies":{"@adobe/node-fetch-retry":"github:adobe/node-fetch-retry","@babel/cli":"^7.12.10","@babel/core":"^7.12.10","avsc":"^5.5.3","xxhashjs":"^0.2.2","babel-preset-gatsby-package":"^0.11.0-next.0","bottleneck":"^2.19.5","execa":"^5.0.0","fastq":"^1.10.0","fs-extra":"^9.0.1","lodash":"^4.17.20","p-queue":"^6.6.2","redis":"^3.0.2","uuid":"^8.3.2","socket.io":"^2.0.0","socket.io-client":"^2.0.0","yargs":"^16.2.0"},"devDependencies":{"cross-env":"^7.0.3","detect-port":"^1.3.0","wait-on":"^5.2.1"},"_id":"dagsby@0.0.3","_nodeVersion":"10.23.2","_npmVersion":"6.14.10","dist":{"shasum":"3f66580229a62082db2d5efd40e49ad3e64d8f27","size":401599,"noattachment":false,"key":"/dagsby/-/dagsby-0.0.3.tgz","tarball":"http://registry.cnpm.dingdandao.com/dagsby/download/dagsby-0.0.3.tgz"},"_npmUser":{"name":"kylemathews","email":"mathews.kyle@gmail.com"},"directories":{},"maintainers":[{"name":"kylemathews","email":""}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/dagsby_0.0.3_1612824511923_0.3508614318512775"},"_hasShrinkwrap":false,"publish_time":1612824512103,"_cnpm_publish_time":1612824512103},"0.0.2":{"name":"dagsby","version":"0.0.2","description":"Gatsby library for orchestrating running data pipelines across workers","main":"dist/index.js","scripts":{"build":"babel src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\"","prepare":"cross-env NODE_ENV=production npm run build","watch":"babel -w src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\""},"author":{"name":"Kyle Mathews","email":"mathews.kyle@gmail.com"},"license":"MIT","dependencies":{"@adobe/node-fetch-retry":"github:adobe/node-fetch-retry","@babel/cli":"^7.12.10","@babel/core":"^7.12.10","avsc":"^5.5.3","xxhashjs":"^0.2.2","babel-preset-gatsby-package":"^0.11.0-next.0","bottleneck":"^2.19.5","execa":"^5.0.0","fastq":"^1.10.0","fs-extra":"^9.0.1","lodash":"^4.17.20","p-queue":"^6.6.2","redis":"^3.0.2","uuid":"^8.3.2","socket.io":"^2.0.0","socket.io-client":"^2.0.0","yargs":"^16.2.0"},"devDependencies":{"cross-env":"^7.0.3","detect-port":"^1.3.0","wait-on":"^5.2.1"},"_id":"dagsby@0.0.2","_nodeVersion":"14.15.3","_npmVersion":"6.14.9","dist":{"shasum":"1fc9afc7411ffd1213ec1e8bbf4a3d2ebfbe0296","size":401680,"noattachment":false,"key":"/dagsby/-/dagsby-0.0.2.tgz","tarball":"http://registry.cnpm.dingdandao.com/dagsby/download/dagsby-0.0.2.tgz"},"_npmUser":{"name":"kylemathews","email":"mathews.kyle@gmail.com"},"directories":{},"maintainers":[{"name":"kylemathews","email":""}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/dagsby_0.0.2_1611604960718_0.13955783356118134"},"_hasShrinkwrap":false,"publish_time":1611604960927,"_cnpm_publish_time":1611604960927},"0.0.1":{"name":"dagsby","version":"0.0.1","description":"Gatsby library for orchestrating running data pipelines across workers","main":"index.js","scripts":{"build":"babel src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\"","prepare":"cross-env NODE_ENV=production npm run build","watch":"babel -w src --out-dir dist --ignore \"**/__tests__\" --extensions \".ts,.js\""},"author":{"name":"Kyle Mathews","email":"mathews.kyle@gmail.com"},"license":"MIT","dependencies":{"@adobe/node-fetch-retry":"github:adobe/node-fetch-retry","@babel/cli":"^7.12.10","@babel/core":"^7.12.10","avsc":"^5.5.3","babel-plugin-remove-graphql-queries":"^2.14.0","babel-preset-gatsby-package":"^0.11.0-next.0","bottleneck":"^2.19.5","execa":"^5.0.0","fastq":"^1.10.0","fs-extra":"^9.0.1","lodash":"^4.17.20","p-queue":"^6.6.2","redis":"^3.0.2","uuid":"^8.3.2","yargs":"^16.2.0"},"devDependencies":{"cross-env":"^7.0.3","detect-port":"^1.3.0","wait-on":"^5.2.1"},"_id":"dagsby@0.0.1","_nodeVersion":"14.15.3","_npmVersion":"6.14.9","dist":{"shasum":"dec63ee36c2d4b16c828504bf7ab61505901edce","size":400923,"noattachment":false,"key":"/dagsby/-/dagsby-0.0.1.tgz","tarball":"http://registry.cnpm.dingdandao.com/dagsby/download/dagsby-0.0.1.tgz"},"_npmUser":{"name":"kylemathews","email":"mathews.kyle@gmail.com"},"directories":{},"maintainers":[{"name":"kylemathews","email":""}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/dagsby_0.0.1_1611603445898_0.2360853267760381"},"_hasShrinkwrap":false,"publish_time":1611603446111,"_cnpm_publish_time":1611603446111}},"readme":"# Dagsby\n\nAn experimental library for orchestrating job running and data processing across processes/machines within Gatsby.\n\n## Components\n\n- **task/job** defines inputs/outputs, dependencies, and code to execute\n- **worker pool** pool of node.js processes which can execute jobs\n- **runner** Uses N worker pools to run tasks.\n\n## Getting started\n\n`npm i dagsby`\n\nStart a worker pool:\n\n```sh\nnode node_modules/dagsby/dist/worker-pool-server.js --numWorkers 4 --socketPort 9999 --httpPort 10020\n```\n\nCreate a simple task in a test.js file and run it on the worker pool.\n\n```js\nconst dagsby = require(`dagsby`)\n\n;(async () => {\n  // Create our runner.\n  const runner = await dagsby.createRunner({\n    pools: [{ socketPort: 9999, httpPort: 10020 }],\n  })\n\n  // Create a simple task\n  const task = await dagsby.createTask({\n    func: args => `Hello ${args.name}!`,\n    // Written using Arvo's schema language.\n    argsSchema: [\n      {\n        name: `name`,\n        type: `string`,\n      },\n    ],\n  })\n\n  // Setup the task on the worker pool(s).\n  await runner.setupTask(task)\n\n  // Run the task!\n  const result = await runner.executeTask({ task, args: { name: `World` } })\n\n  console.log(result)\n})()\n```\n\nLet's try a more complex task where we specify a required file & add an NPM dependency.\n\nFirst create a file called `hello.txt` with some text in it.\n\nThen add this code to our test file after the first task.\n\n```js\nconst mySecondTask = await dagsby.createTask({\n  func: (args, { files }) => {\n    const fs = require(`fs`)\n    const _ = require(`lodash`)\n    const text = fs.readFileSync(files.text.localPath)\n    const camelCase = _.camelCase(text)\n\n    return `${args.preface} ${text} \\n\\n ${camelCase}`\n  },\n  argsSchema: [{ name: `preface`, type: `string` }],\n  dependencies: {\n    lodash: `latest`,\n  },\n  files: {\n    text: {\n      originPath: require(`path`).join(__dirname, `hello.txt`),\n    },\n  },\n})\nawait runner.setupTask(mySecondTask)\n\nconst result2 = await runner.executeTask({\n  task: mySecondTask,\n  args: { preface: `yeeesss` },\n})\n\nconsole.log(result2)\n```\n\n## TODOs\n\n- [ ] support (again) running multiple types of tasks in parallel.\n- [ ] support multiple pools in runners.\n","_attachments":{},"license":"MIT"}