Update file download logic to support checked in copies.

.gitignore

@@ -13,3 +13,5 @@ node_modules/
 /.wwwtfrc
 /service-account-credentials.json
 /secrets.tar
+/schedule.json
+/contents/schedule-json.txt

.travis.yml

@@ -7,17 +7,17 @@ node_js:
 
 script:
   - npm run-script generate-locals
-  # - npm run-script generate-redirects
+  - npm run-script generate-redirects
   - npm run-script build
   # Because we build again, move the non-deploy build out of the way.
-  # - mv build test-build
+   - mv build test-build
 
 before_deploy:
-# - openssl aes-256-cbc -K $encrypted_92a5b1b18fab_key -iv $encrypted_92a5b1b18fab_iv -in secrets.tar.enc -out secrets.tar -d
-# - tar xvf secrets.tar
-# - npm run-script ci:import
-# - npm run-script build
-# - node tests/post-build.js
+  - openssl aes-256-cbc -in secrets.tar.enc -out secrets.tar.out -d -pass "env:encrypted_client_secrets_password"
+  - tar xvf secrets.tar
+  - npm run-script ci:import
+  - npm run-script build
+  - node tests/post-build.js
 
 deploy:
   provider: pages

contents/images/cms/README.md

@@ -0,0 +1 @@
+# Make this exist

plugins/nunjucks.js

@@ -13,7 +13,7 @@ module.exports = function(env, callback) {
     extensions: {}
   };
   Object.assign(env.config.locals, require('../locals-generated.json'));
-  env.config.locals.schedule = require('../schedule.json');
+  // env.config.locals.schedule = require('../schedule.json');
   env.config.locals.Date = Date;
 
   // Load the new nunjucks environment.
@@ -47,14 +47,18 @@ module.exports = function(env, callback) {
   nenv.opts.autoescape = options.autoescape;
 
   class NunjucksTemplate extends env.TemplatePlugin {
-    constructor(template) {
+    constructor(template, filename) {
       super();
       this.template = template;
+      this.filename = filename;
     }
 
     render(locals, callback) {
       try {
         let html = this.template.render(locals);
+        if (!html) {
+          throw new Error('Template render failed' + this.filename);
+        }
         html = minify(html, {
           removeAttributeQuotes: true,
           collapseWhitespace: true,
@@ -69,7 +73,7 @@ module.exports = function(env, callback) {
     }
 
     static fromFile(filepath, callback) {
-      callback(null, new NunjucksTemplate(nenv.getTemplate(filepath.relative)));
+      callback(null, new NunjucksTemplate(nenv.getTemplate(filepath.relative), filepath.relative));
     }
   }
 

scripts/spreadsheet-import/credentials.js

@@ -0,0 +1,86 @@
+const path = require('path');
+const fs = require('fs');
+const credentialsCache = {};
+
+function getCredentialsPath() {
+  return (
+    process.env.WWWTF_CREDENTIALS_PATH ||
+    path.resolve(process.env.HOME, '.wwwtf18')
+  );
+}
+
+/**
+ * Loads a private credentials-file.
+ * Files with sensitive information are stored in the directory ~/.wwwtf18
+ * in JSON-Format. An alternative directory can be specified using the
+ * env-variable `WWWTF_CREDENTIALS_PATH`, but that is mostly intended
+ * to be used for CI/CD-servers and similar environments.
+ * @param {string} filename json-filename without any path.
+ * @param {boolean} ignoreMissing true to ignore missing files and return
+ *   no content.
+ * @returns {*} the parsed content of the json-file
+ */
+function loadCredentials(filename, ignoreMissing = false) {
+  if (!credentialsCache[filename]) {
+    const credentialsFile = path.resolve(getCredentialsPath(), filename);
+
+    if (ignoreMissing && !fs.existsSync(credentialsFile)) {
+      return null;
+    }
+
+    try {
+      credentialsCache[filename] = require(credentialsFile);
+    } catch (err) {
+      console.error(
+        "🔐  It appears that you don't have your credentials setup yet.\n" +
+          '  Please copy the file ' +
+          filename +
+          ' to\n    ' +
+          getCredentialsPath() +
+          '\n  to continue. Ask your coworkers if you never heard of that file.'
+      );
+
+      throw new Error(`failed to load ${credentialsFile}: ${err}`);
+    }
+  }
+
+  return credentialsCache[filename];
+}
+
+/**
+ * Checks if credentials with the given filename exist.
+ * @param {string} filename file basename
+ * @return {boolean} -
+ */
+function hasCredentials(filename) {
+  const credentialsFile = path.resolve(getCredentialsPath(), filename);
+
+  return fs.existsSync(credentialsFile);
+}
+
+/**
+ * Stores credentials to a file in the credentials-store.
+ * @param {string} filename the file basename.
+ * @param {object} data the data to store.
+ */
+function storeCredentials(filename, data) {
+  credentialsCache[filename] = data;
+
+  const credentialsFile = path.resolve(getCredentialsPath(), filename);
+  try {
+    fs.writeFileSync(credentialsFile, JSON.stringify(data));
+  } catch (err) {
+    console.error(
+      `🔐  Failed to write credentials to file ${credentialsFile}.`
+    );
+
+    throw new Error(`failed to write credentials: ${err}`);
+  }
+}
+
+module.exports = {
+  getCredentialsPath,
+  loadCredentials,
+  hasCredentials,
+  storeCredentials
+};

scripts/spreadsheet-import/image-utils/image-download.js

@@ -0,0 +1,98 @@
+const fs = require('fs');
+const path = require('path');
+const fetch = require('node-fetch');
+const chalk = require('chalk');
+const imageType = require('image-type');
+const imageSize = require('image-size');
+const {promisify} = require('util');
+
+function getImageFilename(originalUrl, name, ext) {
+  let filename = name || 'image';
+  filename = filename.replace(/[^\w]/g, '-');
+  filename = filename.replace(/--/g, '-');
+  // Filename changes if underlying URL changes.
+  const hash = require('crypto').createHash('sha1')
+      .update(originalUrl).digest('hex');
+  filename += '-' + hash.substr(0, 8);
+  return (filename + '.' + ext).toLowerCase();
+}
+
+async function existingImage(url, name, opt_extension) {
+  const extensions = ['jpg', 'png', 'gif', 'jpeg'];
+  if (opt_extension) {
+    extensions.unshift(opt_extension);
+  }
+  for (let ext of extensions) {
+    let filename = getImageFilename(url, name, ext);
+    if (await promisify(fs.exists)(fullPath(filename))) {
+      console.info(' --> existing image', chalk.green(fullPath(filename)));
+      return {
+        ext,
+        filename,
+        buffer: await promisify(fs.readFile)(fullPath(filename)),
+      };
+    }
+  }
+  return null;
+}
+
+function fullPath(filename) {
+  return 'contents/images/cms/' + filename;
+}
+
+// Downloads an image from a url unless a local copy is available.
+// name should be any string for use in the filename
+// Pass in opt_extension if you know the type of the image.
+async function downloadImage(url, name, opt_extension) {
+  console.log('Downloading', url);
+  if (!url) {
+    console.error(chalk.yellow('Skipping empty image url'));
+    return {};
+  }
+
+  try {
+    let info = await existingImage(url, name, opt_extension);
+    if (!info) {
+      info = {};
+      const res = await fetch(url);
+      const contentType = res.headers.get('content-type');
+      if (!contentType || !contentType.startsWith('image')) {
+        console.error(chalk.red.bold(' !!! url is not an image', url));
+        return {};
+      }
+
+      const buffer = await res.buffer();
+      info.buffer = buffer;
+
+      const type = opt_extension == 'svg' ? {ext: 'svg'} : imageType(buffer);
+      if (!type) {
+        console.error(chalk.red.bold(' !!! no type-information for image', url));
+        return {};
+      }
+      info.ext = type.ext;
+      const filename = getImageFilename(url, name, info.ext);
+      info.filename = filename;
+
+      const path = fullPath(filename);
+      console.info(' --> image downloaded ', chalk.green(path));
+      fs.writeFile(path, buffer, () => {/*fire and forget*/});
+    }
+    let size = {};
+    try {
+      size = imageSize(info.buffer) || {};
+    } catch (e) {
+      console.error(chalk.yellow('Can\'t get image size' + e.message));
+    }
+    return {
+      filename: info.filename,
+      width: size.width,
+      height: size.height
+    };
+  } catch (err) {
+    console.error(chalk.red.bold(' !!! failed to download', url));
+    console.error(err);
+    return {};
+  }
+}
+
+module.exports = {downloadImage};