[IN-201]: chore: remove storj-lib and storj-mongodb-adapter

lib/core/shards/validateContract.js

@@ -0,0 +1,63 @@
+'use strict';
+
+/**
+ * Contract rules from storj-lib (https://github.com/internxt/core)
+ *
+ * Rules: https://github.com/internxt/core/blob/5879f682cf24e3a1f861e61ff505c89e73ccccb8/lib/contract/schema.json
+ * Error: https://github.com/internxt/core/blob/5879f682cf24e3a1f861e61ff505c89e73ccccb8/lib/contract/index.js#L52
+ */
+
+const assert = require('assert');
+
+const isType = {
+  // https://github.com/bugventure/jsen/blob/9e57a9d653ff4ea47d364fc37da0f1659bfb8e89/lib/jsen.js#L67
+  integer: (value) => typeof value === 'number' && !(value % 1),
+  string: (value) => typeof value === 'string',
+  boolean: (value) => typeof value === 'boolean',
+  null: (value) => value === null,
+};
+
+const rule = (types, { minimum, maximum, pattern } = {}) => (value) => {
+  if (!types.some((type) => isType[type](value))) {
+    return false;
+  }
+  if (value < minimum || value > maximum) {
+    return false;
+  }
+
+  return typeof value !== 'string' || pattern === undefined || pattern.test(value);
+};
+
+const text = rule(['string', 'null']);
+const price = rule(['integer', 'null']);
+const positiveNumber = rule(['integer', 'null'], { minimum: 0 });
+const nodeId = rule(['string', 'null'], { pattern: /[A-Fa-f0-9]{40}$/ });
+
+const CONTRACT_RULES = {
+  version: rule(['integer'], { minimum: 0 }),
+  renter_hd_key: rule(['string', 'boolean'], { pattern: /^[1-9a-km-zA-HJ-NP-Z]{1,111}$/ }),
+  renter_hd_index: rule(['integer', 'boolean'], { minimum: 0, maximum: 2147483647 }),
+  renter_id: nodeId,
+  renter_signature: text,
+  farmer_id: nodeId,
+  farmer_signature: text,
+  data_size: positiveNumber,
+  data_hash: rule(['string', 'null'], { pattern: /^[0-9a-f]{40}$/ }),
+  store_begin: positiveNumber,
+  store_end: positiveNumber,
+  audit_count: positiveNumber,
+  payment_storage_price: price,
+  payment_download_price: price,
+  payment_destination: text,
+};
+
+const validateContract = (contract) => {
+  const properties = Object(contract);
+  const isValid = Object.keys(CONTRACT_RULES).every((field) => (
+    properties[field] === undefined || CONTRACT_RULES[field](properties[field])
+  ));
+
+  assert.ok(isValid, 'Invalid contract specification was supplied');
+};
+
+module.exports = { validateContract };

lib/engine.js

@@ -3,7 +3,6 @@
 const v8 = require('v8');
 const hat = require('hat');
 const async = require('async');
-const storj = require('storj-lib');
 const assert = require('assert');
 const express = require('express');
 const crossorigin = require('cors');
@@ -17,7 +16,6 @@ const Server = require('./server');
 const pow = require('./server/middleware/pow');
 const Mailer = require('inxt-service-mailer');
 const log = require('./logger');
-const MongoDBStorageAdapter = require('storj-mongodb-adapter');
 const NetworkMessageQueue = require('./server/queues/networkQueue');
 const { bindNewRoutes } = require('./server/http');
 const { json } = require('express');
@@ -70,7 +68,6 @@ function Engine(config) {
   assert(config instanceof Config, 'Invalid config supplied');
 
   this._config = config;
-  this._apispec = null;
   this._pendingResponses = {};
   this._cpuUsage = process.cpuUsage();
 }
@@ -122,10 +119,6 @@ Engine.prototype.start = function (callback) {
   );
 
   this.mailer = new Mailer(this._config.mailer);
-  this.contracts = new storj.StorageManager(
-    new MongoDBStorageAdapter(this.storage.connection),
-    { disableReaper: true }
-  );
   this.redis = require('redis').createClient(this._config.redis);
   this.redis.on('ready', () => {
     log.info('connected to redis');
@@ -212,7 +205,6 @@ Engine.prototype._configureApp = function () {
     config: this._config,
     storage: this.storage,
     mailer: this.mailer,
-    contracts: this.contracts,
     redis: this.redis,
     networkQueue: this.networkQueue,
   });
@@ -249,7 +241,6 @@ Engine.prototype._configureApp = function () {
   app.use(this._trackResponseStatus.bind(this));
   app.use(crossorigin(corsOptions));
   app.use(helmet());
-  app.get('/', this._handleRootGET.bind(this));
   routers.forEach(bindRoute);
   app.use(unexpectedErrorLogger);
   app.use(errorHandler({ logger: log }));
@@ -281,14 +272,6 @@ Engine.prototype._configureApp = function () {
   return app;
 };
 
-/**
- * Responds with the swagger spec
- * @private
- */
-Engine.prototype._handleRootGET = function (req, res) {
-  res.send(this.getSpecification());
-};
-
 /**
  * Keeps tabs on all of the pending responses
  * @private
@@ -377,21 +360,4 @@ Engine.prototype._logHealthInfo = function () {
   );
 };
 
-/**
- * Returns a dictionary of info about the service
- * @returns {Object}
- */
-Engine.prototype.getSpecification = function () {
-  this._apispec = require('./apispec.json');
-  this._apispec.schemes = this._config.server.ssl.cert ? ['https'] : ['http'];
-  this._apispec.host = this._config.server.host;
-  this._apispec.info = {
-    title: 'Internxt Bridge',
-    'x-protocol-version': storj.version.protocol,
-    'x-core-version': storj.version.software,
-  };
-
-  return this._apispec;
-};
-
 module.exports = Engine;

lib/server/routefactory.js

@@ -14,7 +14,6 @@ module.exports = function RouteFactory(options) {
       network: options.network,
       storage: options.storage,
       mailer: options.mailer,
-      contracts: options.contracts,
       redis: options.redis,
       networkQueue: options.networkQueue,
     }).getEndpointDefinitions();

lib/server/routes/buckets.js

@@ -3,7 +3,7 @@
 const ms = require('ms');
 const assert = require('assert');
 const async = require('async');
-const storj = require('storj-lib');
+const { validateContract } = require('../../core/shards/validateContract');
 const authenticate = require('../middleware/authenticate');
 const tokenauth = require('../middleware/token-auth');
 const publicBucket = require('../middleware/public-bucket');
@@ -678,6 +678,41 @@ BucketsRouter.prototype.listMirrorsForFile = function (req, res, next) {
   });
 };
 
+const ID_LENGTH = 40;
+
+// Replaces storj Contact: https://github.com/internxt/core/blob/5879f682cf24e3a1f861e61ff505c89e73ccccb8/lib/network/contact.js#L23-L44
+// address, port, nodeID and lastSeen are inherited from kad's AddressPortContact
+const toFarmerContact = (contact) => {
+  assert(typeof contact.address === 'string', 'Invalid address was supplied');
+  assert(typeof contact.port === 'number', 'Invalid port was supplied');
+  assert(!!contact.nodeID && contact.nodeID.length === ID_LENGTH, 'Invalid nodeID was supplied');
+
+  return {
+    userAgent: contact.userAgent || '8.7.2',
+    protocol: contact.protocol || '1.2.0',
+    address: contact.address,
+    port: contact.port,
+    nodeID: contact.nodeID,
+    lastSeen: contact.lastSeen || Date.now(),
+  };
+};
+
+// From storj-mongodb-adapter and storj-lib
+BucketsRouter.prototype._getShardNodeIDs = async function (hash) {
+  const shard = await this.storage.models.Shard.findOne({ hash }).exec();
+
+  if (!shard) {
+    throw new Error('Shard data not found');
+  }
+
+  const { contracts } = shard.toObject();
+  const nodeIDs = Object.keys(contracts);
+
+  nodeIDs.forEach((nodeID) => validateContract(contracts[nodeID]));
+
+  return nodeIDs;
+};
+
 /**
  * Fetches a RETRIEVE token from a farmer for the given shard
  * @private
@@ -690,12 +725,12 @@ BucketsRouter.prototype._getRetrievalToken = function (sPointer, opts, done) {
   const self = this;
 
   log.debug('getting retrieval token for %j', sPointer);
-  this.contracts.load(sPointer.hash, async function (err, item) {
-    if (err) {
-      return done(err);
-    }
+  // From storj-lib
+  assert(typeof sPointer.hash === 'string', 'Invalid key supplied');
+  assert(sPointer.hash.length === ID_LENGTH, 'Key must be 160 bit hex string');
 
-    let farmers = Object.keys(item.contracts).filter((nodeID) => {
+  self._getShardNodeIDs(sPointer.hash).then(async (nodeIDs) => {
+    let farmers = nodeIDs.filter((nodeID) => {
       return opts.excludeFarmers.indexOf(nodeID) === -1;
     });
 
@@ -717,7 +752,7 @@ BucketsRouter.prototype._getRetrievalToken = function (sPointer, opts, done) {
           return currentTime - c.lastTimeout > farmerTimeoutMs;
         })
         .sort((a, b) => b.lastSeen - a.lastSeen)
-        .map((c) => ({ contact: storj.Contact(c), pointer: null }));
+        .map((contact) => ({ contact: toFarmerContact(contact), pointer: null }));
 
       let retrievalTimeout = setTimeout(() => {
         handleResults(new errors.ServiceUnavailableError('Timed out waiting for pointers'));
@@ -755,13 +790,13 @@ BucketsRouter.prototype._getRetrievalToken = function (sPointer, opts, done) {
       async.detectLimit(
         options,
         6,
-        self._requestRetrievalPointer.bind(self, item),
+        self._requestRetrievalPointer.bind(self),
         handleResults
       );
     } catch (err) {
       return done(err);
     }
-  });
+  }, done);
 };
 /**
  * @callback BucketsRouter~_getRetrievalTokenCallback
@@ -777,14 +812,12 @@ BucketsRouter.prototype._getRetrievalToken = function (sPointer, opts, done) {
 /**
  * Requests a retrieval pointer from the first farmer in the given list
  * @private
- * @param {StorageItem} item - Loaded storage item data
  * @param {Object} opts
  * @param {Contact} opts.contact
  * @param {Pointer|null} opts.pointer
  * @param {BucketsRouter~_requestRetrievalPointerCallback}
  */
-BucketsRouter.prototype._requestRetrievalPointer = function (item, meta, done) {
-
+BucketsRouter.prototype._requestRetrievalPointer = function (meta, done) {
   meta.pointer = {
     token: randomBytes(16).toString('hex'),
     farmer: meta.contact

lib/server/routes/index.js

@@ -17,7 +17,6 @@ function Router(options) {
   this.config = options.config;
   this.storage = options.storage;
   this.mailer = options.mailer;
-  this.contracts = options.contracts;
   this.redis = options.redis;
   this.networkQueue = options.networkQueue;
 }

lib/server/routes/users.js

@@ -8,7 +8,6 @@ const merge = require('merge');
 const inherits = require('util').inherits;
 const authenticate = require('../middleware/authenticate');
 const crypto = require('crypto');
-const storj = require('storj-lib');
 const limiter = require('../limiter').DEFAULTS;
 const rateLimiter = require('../middleware/rate-limiter');
 const disposable = require('disposable-email');
@@ -52,7 +51,7 @@ UsersRouter.prototype._dispatchActivationEmail = function (user, redir, cb) {
   // let self = this;
   // let profile = self.config.server.public || self.config.server;
   // let host = profile.host;
-  let callback = cb || storj.utils.noop;
+  let callback = cb || (() => {});
   /*
   let port = [443, 80].indexOf(profile.port) === -1 ?
     ':' + profile.port :

package.json

@@ -68,6 +68,7 @@
     "@types/sinon": "^10.0.11",
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^8.3.4",
+    "bitcore-lib": "8.25.10",
     "chai": "^4.2.0",
     "chance": "^1.1.11",
     "coveralls": "^2.11.6",
@@ -100,6 +101,7 @@
     "amqplib": "^0.8.0",
     "analytics-node": "^2.4.0",
     "async": "^3.2.0",
+    "aws-sdk": "^2.920.0",
     "axios": "0.33.0",
     "basic-auth": "^2.0.1",
     "bn.js": "^4.11.8",
@@ -121,9 +123,7 @@
     "redis": "^3.1.0",
     "secp256k1": "^4.0.2",
     "semver": "^7.7.4",
-    "storj-lib": "github:internxt/core#v8.7.3-beta",
-    "storj-mongodb-adapter": "github:internxt/mongodb-adapter#10.1.0-beta",
-    "storj-service-storage-models": "github:internxt/service-storage-models#11.1.0",
+    "storj-service-storage-models": "github:internxt/service-storage-models#11.2.0",
     "stripe": "^8.49.0",
     "through": "^2.3.8",
     "typescript": "^4.6.3",

tests/lib/core/shards/validateContract.test.ts

@@ -0,0 +1,37 @@
+import { validateContract } from '../../../../lib/core/shards/validateContract';
+
+describe('validateContract()', () => {
+  it('When a contract only has valid values, then it should not throw', () => {
+    const contract = {
+      version: 0,
+      renter_id: 'a'.repeat(40),
+      data_hash: 'b'.repeat(40),
+      data_size: 1024,
+      store_begin: 1,
+      store_end: 2,
+      renter_hd_key: false,
+    };
+
+    expect(() => validateContract(contract)).not.toThrow();
+  });
+
+  it('When a contract has unknown fields, then it should ignore them', () => {
+    expect(() => validateContract({ unknown: 'value' })).not.toThrow();
+  });
+
+  it('When a contract is empty, then it should be valid', () => {
+    expect(() => validateContract(undefined)).not.toThrow();
+  });
+
+  it.each([
+    ['store_end', 'invalid'],
+    ['store_begin', -1],
+    ['data_size', 1.5],
+    ['data_hash', 'A'.repeat(40)],
+    ['renter_id', 'not-hex'],
+    ['renter_hd_index', 2147483648],
+    ['version', null],
+  ])('When %s has the value %p, then it should throw the contract error', (field, value) => {
+    expect(() => validateContract({ [field]: value })).toThrow('Invalid contract specification was supplied');
+  });
+});