feat: Kernel module

package.json

@@ -42,7 +42,7 @@
     "bcryptjs-then": "1.0.1",
     "bluebird": "3.5.1",
     "body-parser": "1.18.2",
-    "bull": "3.2.0",
+    "bull": "3.3.0",
     "bunyan": "1.8.12",
     "cheerio": "1.0.0-rc.2",
     "child-process-promise": "2.2.1",
@@ -66,7 +66,7 @@
     "graphql": "0.10.5",
     "graphql-tools": "2.2.1",
     "highlight.js": "9.12.0",
-    "i18next": "9.0.1",
+    "i18next": "9.1.0",
     "i18next-express-middleware": "1.0.7",
     "i18next-localstorage-cache": "1.1.1",
     "i18next-node-fs-backend": "1.0.0",
@@ -76,7 +76,6 @@
     "js-yaml": "3.10.0",
     "jsonwebtoken": "8.0.1",
     "klaw": "2.1.0",
-    "levelup": "1.3.9",
     "lodash": "4.17.4",
     "markdown-it": "8.4.0",
     "markdown-it-abbr": "1.0.4",
@@ -89,7 +88,6 @@
     "markdown-it-mathjax": "2.0.0",
     "markdown-it-task-lists": "2.0.1",
     "mathjax-node": "1.2.1",
-    "memdown": "1.4.1",
     "mime-types": "2.1.17",
     "moment": "2.18.1",
     "moment-timezone": "0.5.13",
@@ -106,21 +104,19 @@
     "passport-local": "1.0.0",
     "passport-slack": "0.0.7",
     "passport-windowslive": "1.0.2",
-    "pg": "7.3.0",
+    "pg": "6.4.2",
     "pg-hstore": "2.3.2",
     "pg-promise": "6.10.3",
     "pm2": "2.7.1",
     "pug": "2.0.0-rc.4",
+    "qr-image": "3.2.0",
     "read-chunk": "2.1.0",
     "remove-markdown": "0.2.2",
     "request": "2.83.0",
-    "search-index-adder": "0.3.9",
-    "search-index-searcher": "0.2.10",
     "semver": "5.4.1",
     "sequelize": "4.13.5",
     "serve-favicon": "2.4.5",
     "simplemde": "1.11.2",
-    "stopword": "0.1.8",
     "stream-to-promise": "2.2.0",
     "tar": "4.0.1",
     "through2": "2.0.3",
@@ -135,6 +131,7 @@
     "apollo-client": "^1.9.3",
     "autoprefixer": "7.1.5",
     "babel-cli": "6.26.0",
+    "babel-core": "6.26.0",
     "babel-jest": "21.2.0",
     "babel-preset-env": "1.6.0",
     "babel-preset-es2015": "6.24.1",

server/index.js

@@ -1,5 +1,3 @@
-'use strict'
-
 // ===========================================
 // Wiki.js
 // Licensed under AGPLv3
@@ -13,7 +11,8 @@ let wiki = {
   IS_MASTER: cluster.isMaster,
   ROOTPATH: process.cwd(),
   SERVERPATH: path.join(process.cwd(), 'server'),
-  configSvc: require('./modules/config')
+  configSvc: require('./modules/config'),
+  kernel: require('./modules/kernel')
 }
 global.wiki = wiki
 
@@ -38,37 +37,7 @@ wiki.logger = require('./modules/logger').init()
 wiki.db = require('./modules/db').init()
 
 // ----------------------------------------
-// Start Cluster
+// Start Kernel
 // ----------------------------------------
 
-const numCPUs = require('os').cpus().length
+wiki.kernel.init()
-let numWorkers = (wiki.config.workers > 0) ? wiki.config.workers : numCPUs
-if (numWorkers > numCPUs) {
-  numWorkers = numCPUs
-}
-
-if (cluster.isMaster) {
-  wiki.logger.info('=======================================')
-  wiki.logger.info('= Wiki.js =============================')
-  wiki.logger.info('=======================================')
-
-  require('./master').then(() => {
-    // -> Create background workers
-    for (let i = 0; i < numWorkers; i++) {
-      cluster.fork()
-    }
-
-    // -> Queue post-init tasks
-
-    wiki.queue.uplClearTemp.add({}, {
-      repeat: { cron: '*/15 * * * *' }
-    })
-  })
-
-  cluster.on('exit', (worker, code, signal) => {
-    wiki.logger.info(`Background Worker #${worker.id} was terminated.`)
-  })
-} else {
-  wiki.logger.info(`Background Worker #${cluster.worker.id} is initializing...`)
-  require('./worker')
-}

server/init.js

@@ -1,90 +0,0 @@
-'use strict'
-
-const Promise = require('bluebird')
-const fs = Promise.promisifyAll(require('fs-extra'))
-const pm2 = Promise.promisifyAll(require('pm2'))
-const ora = require('ora')
-const path = require('path')
-
-const ROOTPATH = process.cwd()
-
-module.exports = {
-  /**
-   * Detect the most appropriate start mode
-   */
-  startDetect: function () {
-    if (process.env.WIKI_JS_HEROKU) {
-      return this.startInHerokuMode()
-    } else {
-      return this.startInBackgroundMode()
-    }
-  },
-  /**
-   * Start in background mode
-   */
-  startInBackgroundMode: function () {
-    let spinner = ora('Initializing...').start()
-    return fs.emptyDirAsync(path.join(ROOTPATH, './logs')).then(() => {
-      return pm2.connectAsync().then(() => {
-        return pm2.startAsync({
-          name: 'wiki',
-          script: 'server',
-          cwd: ROOTPATH,
-          output: path.join(ROOTPATH, './logs/wiki-output.log'),
-          error: path.join(ROOTPATH, './logs/wiki-error.log'),
-          minUptime: 5000,
-          maxRestarts: 5
-        }).then(() => {
-          spinner.succeed('Wiki.js has started successfully.')
-        }).finally(() => {
-          pm2.disconnect()
-        })
-      })
-    }).catch(err => {
-      spinner.fail(err)
-      process.exit(1)
-    })
-  },
-  /**
-   * Start in Heroku mode
-   */
-  startInHerokuMode: function () {
-    console.warn('Incorrect command on Heroku, use instead: node server')
-    process.exit(1)
-  },
-  /**
-   * Stop Wiki.js process(es)
-   */
-  stop () {
-    let spinner = ora('Shutting down Wiki.js...').start()
-    return pm2.connectAsync().then(() => {
-      return pm2.stopAsync('wiki').then(() => {
-        spinner.succeed('Wiki.js has stopped successfully.')
-      }).finally(() => {
-        pm2.disconnect()
-      })
-    }).catch(err => {
-      spinner.fail(err)
-      process.exit(1)
-    })
-  },
-  /**
-   * Restart Wiki.js process(es)
-   */
-  restart: function () {
-    let self = this
-    return self.stop().delay(1000).then(() => {
-      self.startDetect()
-    })
-  },
-  /**
-   * Start the web-based configuration wizard
-   *
-   * @param {Number} port Port to bind the HTTP server on
-   */
-  configure (port) {
-    port = port || 3000
-    let spinner = ora('Initializing interactive setup...').start()
-    require('./configure')(port, spinner)
-  }
-}

server/master.js

@@ -1,17 +1,6 @@
-'use strict'
-
 /* global wiki */
 
-const Promise = require('bluebird')
+module.exports = () => {
-
-wiki.redis = require('./modules/redis').init()
-wiki.queue = require('./modules/queue').init()
-
-module.exports = Promise.join(
-  wiki.db.onReady,
-  wiki.configSvc.loadFromDb(),
-  wiki.queue.clean()
-).then(() => {
   // ----------------------------------------
   // Load global modules
   // ----------------------------------------
@@ -194,7 +183,4 @@ module.exports = Promise.join(
   })
 
   return true
-}).catch(err => {
+}
-  wiki.logger.error(err)
-  process.exit(1)
-})

server/models/user.js

@@ -19,7 +19,7 @@ module.exports = (sequelize, DataTypes) => {
       }
     },
     provider: {
-      type: DataTypes.ENUM(wiki.data.authProviders),
+      type: DataTypes.STRING,
       allowNull: false
     },
     providerId: {
@@ -37,6 +37,15 @@ module.exports = (sequelize, DataTypes) => {
     role: {
       type: DataTypes.ENUM('admin', 'user', 'guest'),
       allowNull: false
+    },
+    tfaIsActive: {
+      type: DataTypes.BOOLEAN,
+      allowNull: false,
+      defaultValue: false
+    },
+    tfaSecret: {
+      type: DataTypes.STRING,
+      allowNull: true
     }
   }, {
     timestamps: true,

server/modules/config.js

@@ -75,13 +75,14 @@ module.exports = {
         }
       }
     }).then(results => {
-      if (_.isArray(results) && results.length > 0) {
+      if (_.isArray(results) && results.length === subsets.length) {
         results.forEach(result => {
           wiki.config[result.key] = result.config
         })
         return true
       } else {
-        return Promise.reject(new Error('Invalid DB Configuration result set'))
+        wiki.logger.warn('DB Configuration is empty or incomplete.')
+        return false
       }
     })
   }

server/modules/db.js

@@ -75,7 +75,7 @@ module.exports = {
         min: 0,
         idle: 10000
       },
-      logging: false,
+      logging: log => { wiki.logger.log('verbose', log) },
       operatorsAliases
     })
 
@@ -92,10 +92,10 @@ module.exports = {
 
     fs
       .readdirSync(dbModelsPath)
-      .filter(function (file) {
+      .filter(file => {
         return (file.indexOf('.') !== 0 && file.indexOf('_') !== 0)
       })
-      .forEach(function (file) {
+      .forEach(file => {
         let modelName = _.upperFirst(_.camelCase(_.split(file, '.')[0]))
         self[modelName] = self.inst.import(path.join(dbModelsPath, file))
       })
@@ -110,8 +110,8 @@ module.exports = {
       // -> Sync DB Schemas
       syncSchemas() {
         return self.inst.sync({
-          force: false,
+          force: true,
-          logging: false
+          logging: log => { wiki.logger.log('verbose', log) }
         })
       },
       // -> Set Connection App Name
@@ -129,7 +129,7 @@ module.exports = {
 
     // Perform init tasks
 
-    self.onReady = Promise.each(initTasksQueue, t => t())
+    self.onReady = Promise.each(initTasksQueue, t => t()).return(true)
 
     return self
   }

server/modules/kernel.js

@@ -0,0 +1,91 @@
+const cluster = require('cluster')
+const Promise = require('bluebird')
+const _ = require('lodash')
+
+/* global wiki */
+
+module.exports = {
+  numWorkers: 1,
+  workers: [],
+  init() {
+    if (cluster.isMaster) {
+      wiki.logger.info('=======================================')
+      wiki.logger.info('= Wiki.js =============================')
+      wiki.logger.info('=======================================')
+
+      wiki.redis = require('./redis').init()
+      wiki.queue = require('./queue').init()
+
+      this.setWorkerLimit()
+      this.bootMaster()
+    } else {
+      this.bootWorker()
+    }
+  },
+  /**
+   * Pre-Master Boot Sequence
+   */
+  preBootMaster() {
+    return Promise.mapSeries([
+      () => { return wiki.db.onReady },
+      () => { return wiki.configSvc.loadFromDb() },
+      () => { return wiki.queue.clean() }
+    ], fn => { return fn() })
+  },
+  /**
+   * Boot Master Process
+   */
+  bootMaster() {
+    this.preBootMaster().then(sequenceResults => {
+      if (_.every(sequenceResults, rs => rs === true)) {
+        this.postBootMaster()
+      } else {
+        wiki.logger.info('Starting configuration manager...')
+        require('../configure')()
+      }
+      return true
+    }).catch(err => {
+      wiki.logger.error(err)
+      process.exit(1)
+    })
+  },
+  /**
+   * Post-Master Boot Sequence
+   */
+  postBootMaster() {
+    require('../master')().then(() => {
+      _.times(this.numWorker, this.spawnWorker)
+
+      wiki.queue.uplClearTemp.add({}, {
+        repeat: { cron: '*/15 * * * *' }
+      })
+    })
+
+    cluster.on('exit', (worker, code, signal) => {
+      wiki.logger.info(`Background Worker #${worker.id} was terminated.`)
+    })
+  },
+  /**
+   * Boot Worker Process
+   */
+  bootWorker() {
+    wiki.logger.info(`Background Worker #${cluster.worker.id} is initializing...`)
+    require('../worker')
+  },
+  /**
+   * Spawn new Worker process
+   */
+  spawnWorker() {
+    this.workers.push(cluster.fork())
+  },
+  /**
+   * Set Worker count based on config + system capabilities
+   */
+  setWorkerLimit() {
+    const numCPUs = require('os').cpus().length
+    this.numWorkers = (wiki.config.workers > 0) ? wiki.config.workers : numCPUs
+    if (this.numWorkers > numCPUs) {
+      this.numWorkers = numCPUs
+    }
+  }
+}

server/modules/queue.js

@@ -30,7 +30,7 @@ module.exports = {
       })
     }).then(() => {
       wiki.logger.info('Purging old queue jobs: OK')
-    }).catch(err => {
+    }).return(true).catch(err => {
       wiki.logger.error(err)
     })
   }

server/worker.js

@@ -1,5 +1,3 @@
-'use strict'
-
 /* global wiki */
 
 const Promise = require('bluebird')

tools/fuse_tasks.js

@@ -65,27 +65,6 @@ module.exports = Promise.mapSeries([
       }
     })
   },
-  /**
-   * i18n
-   */
-  () => {
-    console.info(colors.white('  └── ') + colors.green('Copying i18n client files...'))
-    return fs.ensureDirAsync('./assets/js/i18n').then(() => {
-      return fs.readJsonAsync('./server/locales/en/browser.json').then(enContent => {
-        return fs.readdirAsync('./server/locales').then(langs => {
-          return Promise.map(langs, lang => {
-            console.info(colors.white('      ' + lang + '.json'))
-            let outputPath = path.join('./assets/js/i18n', lang + '.json')
-            return fs.readJsonAsync(path.join('./server/locales', lang + 'browser.json'), 'utf8').then((content) => {
-              return fs.outputJsonAsync(outputPath, _.defaultsDeep(content, enContent))
-            }).catch(err => { // eslint-disable-line handle-callback-err
-              return fs.outputJsonAsync(outputPath, enContent)
-            })
-          })
-        })
-      })
-    })
-  },
   /**
    * Delete Fusebox cache
    */

wiki.js

@@ -1,13 +1,72 @@
 #!/usr/bin/env node
-'use strict'
 
 // ===========================================
 // Wiki.js
-// 1.0.1
+// 2.0
 // Licensed under AGPLv3
 // ===========================================
 
-const init = require('./server/init')
+const Promise = require('bluebird')
+const fs = Promise.promisifyAll(require('fs-extra'))
+const pm2 = Promise.promisifyAll(require('pm2'))
+const ora = require('ora')
+const path = require('path')
+
+const ROOTPATH = process.cwd()
+
+const init = {
+  /**
+   * Start in background mode
+   */
+  start () {
+    let spinner = ora('Initializing...').start()
+    return fs.emptyDirAsync(path.join(ROOTPATH, './logs')).then(() => {
+      return pm2.connectAsync().then(() => {
+        return pm2.startAsync({
+          name: 'wiki',
+          script: 'server',
+          cwd: ROOTPATH,
+          output: path.join(ROOTPATH, './logs/wiki-output.log'),
+          error: path.join(ROOTPATH, './logs/wiki-error.log'),
+          minUptime: 5000,
+          maxRestarts: 5
+        }).then(() => {
+          spinner.succeed('Wiki.js has started successfully.')
+        }).finally(() => {
+          pm2.disconnect()
+        })
+      })
+    }).catch(err => {
+      spinner.fail(err)
+      process.exit(1)
+    })
+  },
+  /**
+   * Stop Wiki.js process(es)
+   */
+  stop () {
+    let spinner = ora('Shutting down Wiki.js...').start()
+    return pm2.connectAsync().then(() => {
+      return pm2.stopAsync('wiki').then(() => {
+        spinner.succeed('Wiki.js has stopped successfully.')
+      }).finally(() => {
+        pm2.disconnect()
+      })
+    }).catch(err => {
+      spinner.fail(err)
+      process.exit(1)
+    })
+  },
+  /**
+   * Restart Wiki.js process(es)
+   */
+  restart: function () {
+    let self = this
+    return self.stop().delay(1000).then(() => {
+      self.startDetect()
+    })
+  }
+}
 
 require('yargs') // eslint-disable-line no-unused-expressions
   .usage('Usage: node $0 <cmd> [args]')
@@ -16,7 +75,7 @@ require('yargs') // eslint-disable-line no-unused-expressions
     alias: ['boot', 'init'],
     desc: 'Start Wiki.js process',
     handler: argv => {
-      init.startDetect()
+      init.start()
     }
   })
   .command({
@@ -35,18 +94,9 @@ require('yargs') // eslint-disable-line no-unused-expressions
       init.restart()
     }
   })
-  .command({
-    command: 'configure [port]',
-    alias: ['config', 'conf', 'cfg', 'setup'],
-    desc: 'Configure Wiki.js using the web-based setup wizard',
-    builder: (yargs) => yargs.default('port', 3000),
-    handler: argv => {
-      init.configure(argv.port)
-    }
-  })
   .recommendCommands()
   .demandCommand(1, 'You must provide one of the accepted commands above.')
   .help()
   .version()
-  .epilogue('Read the docs at https://wiki.requarks.io')
+  .epilogue('Read the docs at https://docs.requarks.io/wiki')
   .argv