Message links are modified to add tracking

helpers.js

@@ -2,15 +2,20 @@
 var mongoose = require('mongoose');
 var nodemailer = require("nodemailer");
 var chalk = require("chalk");
+const cheerio = require('cheerio');
 
-var SentMail = require('./models/sent-mail');
+var db = require("kairoscope-db-models");
+var SentMail = db.SentMail;
 
-var send_mail = function(msg_data) {
+var send_mail = function(msg_data, id) {
+    var content = add_link_tracking(msg_data.message, id)
+    content = add_tracking_pixel(content, id);
     var mail_msg = {
+        id: id,
         to: msg_data.to,
         from: msg_data.from,
         subject: msg_data.subject,
-        html: msg_data.message,
+        html: content,
     }
     // create Nodemailer SES transporter
     var transporter = nodemailer.createTransport({
@@ -28,7 +33,7 @@ var send_mail = function(msg_data) {
             console.log(err);
             save_sent_email(mail_msg, undefined, false, false);
         } else {
-            console.log(`Sent email ${chalk.italic(`\'${info.messageId}\'`)} successfully`);
+            console.log(`Sent email ${chalk.italic(`\'${mail_msg.id}\'`)} successfully`);
             //console.log(info);
             save_sent_email(mail_msg, info, true, true);
         }
@@ -37,10 +42,11 @@ var send_mail = function(msg_data) {
 
 var save_sent_email = function(email_msg, response_data, sent, delivered) {
     var new_sent_mail = new SentMail();
+    if(response_data != undefined) new_sent_mail.sent_id = response_data.messageId;
+    new_sent_mail.message_id = email_msg.id;
     new_sent_mail.to = email_msg.to;
     new_sent_mail.from = email_msg.from;
     new_sent_mail.subject = email_msg.subject;
-    if(response_data != undefined) new_sent_mail.message_id = response_data.messageId;
     new_sent_mail.message = email_msg.html;
     new_sent_mail.sent = sent;
     new_sent_mail.delivered = delivered;
@@ -73,28 +79,34 @@ var check_env_variables = function() {
             console.log(`${e} AWS SES needs a password. Add ${v} to the environment variables.`);
         }
     }
+    var has_tracking_url = env.TRACKING_URL != "" && env.TRACKING_URL != undefined;
+    if(has_tracking_url) {
+        console.log(`Tracking URL: ${chalk.green(`\'${env.TRACKING_URL}\'`)}`);
+    } else {
+        var v = `${chalk.yellow(`\'TRACKING_URL\'`)}`;
+        console.log(`${e} The mail delivery service needs a tracking server URL. Add ${v} to the environment variables.`);
+    }
 }
 
-var start_database = function() {
-    return new Promise((resolve, reject) => {
-        //Set up default mongoose connection
-        var mongoDB = 'mongodb://127.0.0.1/kairoscope_dev';
-        mongoose.connect(mongoDB, {useNewUrlParser: true, useUnifiedTopology: true}).then(
-            () => { 
-                /** ready to use. The `mongoose.connect()` promise resolves to mongoose instance. */ 
-                console.log("Database connected");
-                resolve();
-            },
-            err => { 
-                /** handle initial connection error */ 
-                console.log("Error connecting to database");
-                reject(err);
-            }
-        );
-    });
+var add_tracking_pixel = function(content, id) {
+    var has_tracking_url = process.env.TRACKING_URL != "" && process.env.TRACKING_URL != undefined;
+    if(has_tracking_url == false) return content;
+    var s = `\n<img src="${process.env.TRACKING_URL}${id}/image.png">`;
+    return content + s;
+}
+
+var add_link_tracking = function(content, id) {
+    const html = cheerio.load(content);
+    html('a').each( (index, link) => {
+        var url = html(link).attr('href');
+        var link_id = html(link).text();
+        //link_id = link_id.replaceAll(" ", "-").replaceAll("/", "").replaceAll("\"", "").replaceAll(":", "").replaceAll("'", "");
+        var new_url = `${process.env.TRACKING_URL}${id}/redirect?url=${url}&link=${link_id}`;
+        html(link).attr("href", new_url);
+     });
+     return html.html();
 }
 
 module.exports.send_mail = send_mail;
 module.exports.save_sent_email = save_sent_email;
-module.exports.check_env_variables = check_env_variables;
-module.exports.start_database = start_database;
+module.exports.check_env_variables = check_env_variables;

index.js

@@ -4,51 +4,57 @@ app.use(express.json())
 const port = 3103
 
 var chalk = require("chalk");
+var uniqid = require('uniqid'); 
+var db = require("kairoscope-db-models");
 
 var helpers = require('./helpers');
-
 helpers.check_env_variables();
-helpers.start_database();
 
+db.start().then(() => {
 
-app.get('/', (req, res) => {
-   res.send("Mail Delivery Service");
-})
+    app.get('/', (req, res) => {
+        res.send("Mail Delivery Service");
+    })
 
-app.post('/send', (req, res) => {
-    if(req.body == undefined){
-        console.log(`Received delivery request for message with no data. Aborting`);
-        res.status(400).json({ error: 'No data'}).end();
-        return;
-    }
-    if(req.body.from == "" || req.body.from == undefined){
-        console.log(`Received delivery request for message with no from email. Aborting`);
-        res.status(400).json({ error: 'Missing \'from\' field'}).end();
-        return;
-    }
-    if(req.body.to == "" || req.body.to == undefined){
-        console.log(`Received delivery request for message with no to email. Aborting`);
-        res.status(400).json({ error: 'Missing \'to\' field'}).end();
-        return;
-    }
-    if(req.body.message == "" || req.body.message == undefined){
-        console.log(`Received delivery request for message with no to message. Aborting`);
-        res.status(400).json({ error: 'Missing \'message\' field'}).end();
-        return;
-    }
-    if(req.body.subject == "" || req.body.subject == undefined){
-        console.log(`Received delivery request for message with empty subject. Aborting`);
-        res.status(400).json({ error: 'Missing \'subject\' field'}).end();
-        return;
-    } else {
+    app.post('/send', (req, res) => {
+        if(req.body == undefined){
+            console.log(`Received delivery request for message with no data. Aborting`);
+            res.status(400).json({ error: 'No data'}).end();
+            return;
+        }
+        if(req.body.from == "" || req.body.from == undefined){
+            console.log(`Received delivery request for message with no from email. Aborting`);
+            res.status(400).json({ error: 'Missing \'from\' field'}).end();
+            return;
+        }
+        if(req.body.to == "" || req.body.to == undefined){
+            console.log(`Received delivery request for message with no to email. Aborting`);
+            res.status(400).json({ error: 'Missing \'to\' field'}).end();
+            return;
+        }
+        if(req.body.message == "" || req.body.message == undefined){
+            console.log(`Received delivery request for message with no to message. Aborting`);
+            res.status(400).json({ error: 'Missing \'message\' field'}).end();
+            return;
+        }
+        if(req.body.subject == "" || req.body.subject == undefined){
+            console.log(`Received delivery request for message with empty subject. Aborting`);
+            res.status(400).json({ error: 'Missing \'subject\' field'}).end();
+            return;
+        }
         var subject = chalk.italic(`\'${req.body.subject}\'`);
-        console.log(`Received delivery request for message ${subject}`);
-    }
-    //console.log(req.body);
-    res.status(200).end();
-    helpers.send_mail(req.body);
-})
+        var to = chalk.italic(`\'${req.body.to}\'`);
+        var from = chalk.italic(`\'${req.body.from}\'`);
+        var id = uniqid();
+        var id_text = chalk.italic(`\'${id}\'`);
+        console.log(`Delivering email ${subject} (${id_text}) to ${to} from ${from}`);
+        //console.log(req.body);
+        res.status(200).end();
+        helpers.send_mail(req.body, id);
+    })
+
+    app.listen(port, () => {
+    console.log(`Mail Delivery Service listening at ${chalk.cyan(`http://localhost:${port}`)}`);
+    })
 
-app.listen(port, () => {
-  console.log(`Mail Delivery Service listening at ${chalk.cyan(`http://localhost:${port}`)}`);
-})
+});

models/sent-mail.js

@@ -5,6 +5,7 @@ var Schema = mongoose.Schema;
 var SentMailSchema = new Schema(
   {
     message_id: {type: String, required: true, maxLength: 250},
+    sent_id: {type: String, required: true, maxLength: 250},
     to: {type: String, required: true, maxLength: 150},
     from: {type: String, required: true, maxLength: 150},
     subject: {type: String, required: true, maxLength: 998},
@@ -14,6 +15,7 @@ var SentMailSchema = new Schema(
     sent: { type: Boolean, default: true },
     delivered: { type: Boolean, default: true },
     bounced: { type: Boolean, default: false },
+    opened: { type: Boolean, default: false },
   }
 );
 

package-lock.json

@@ -67,6 +67,11 @@
         "type-is": "~1.6.17"
       }
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
     "bson": {
       "version": "4.5.2",
       "resolved": "https://registry.npmjs.org/bson/-/bson-4.5.2.tgz",
@@ -98,6 +103,32 @@
         "supports-color": "^7.1.0"
       }
     },
+    "cheerio": {
+      "version": "1.0.0-rc.10",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.10.tgz",
+      "integrity": "sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==",
+      "requires": {
+        "cheerio-select": "^1.5.0",
+        "dom-serializer": "^1.3.2",
+        "domhandler": "^4.2.0",
+        "htmlparser2": "^6.1.0",
+        "parse5": "^6.0.1",
+        "parse5-htmlparser2-tree-adapter": "^6.0.1",
+        "tslib": "^2.2.0"
+      }
+    },
+    "cheerio-select": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-1.5.0.tgz",
+      "integrity": "sha512-qocaHPv5ypefh6YNxvnbABM07KMxExbtbfuJoIie3iZXX1ERwYmJcIiRrr9H05ucQP1k28dav8rpdDgjQd8drg==",
+      "requires": {
+        "css-select": "^4.1.3",
+        "css-what": "^5.0.1",
+        "domelementtype": "^2.2.0",
+        "domhandler": "^4.2.0",
+        "domutils": "^2.7.0"
+      }
+    },
     "color-convert": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -134,6 +165,23 @@
       "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
       "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
     },
+    "css-select": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz",
+      "integrity": "sha512-gT3wBNd9Nj49rAbmtFHj1cljIAOLYSX1nZ8CB7TBO3INYckygm5B7LISU/szY//YmdiSLbJvDLOx9VnMVpMBxA==",
+      "requires": {
+        "boolbase": "^1.0.0",
+        "css-what": "^5.0.0",
+        "domhandler": "^4.2.0",
+        "domutils": "^2.6.0",
+        "nth-check": "^2.0.0"
+      }
+    },
+    "css-what": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-5.0.1.tgz",
+      "integrity": "sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg=="
+    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -157,6 +205,39 @@
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
       "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
     },
+    "dom-serializer": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.3.2.tgz",
+      "integrity": "sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig==",
+      "requires": {
+        "domelementtype": "^2.0.1",
+        "domhandler": "^4.2.0",
+        "entities": "^2.0.0"
+      }
+    },
+    "domelementtype": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.2.0.tgz",
+      "integrity": "sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A=="
+    },
+    "domhandler": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.2.2.tgz",
+      "integrity": "sha512-PzE9aBMsdZO8TK4BnuJwH0QT41wgMbRzuZrHUcpYncEjmQazq8QEaBWgLG7ZyC/DAZKEgglpIA6j4Qn/HmxS3w==",
+      "requires": {
+        "domelementtype": "^2.2.0"
+      }
+    },
+    "domutils": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz",
+      "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==",
+      "requires": {
+        "dom-serializer": "^1.0.1",
+        "domelementtype": "^2.2.0",
+        "domhandler": "^4.2.0"
+      }
+    },
     "ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -167,6 +248,11 @@
       "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
     },
+    "entities": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
+      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
+    },
     "escape-html": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -243,6 +329,17 @@
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
     },
+    "htmlparser2": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz",
+      "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==",
+      "requires": {
+        "domelementtype": "^2.0.1",
+        "domhandler": "^4.0.0",
+        "domutils": "^2.5.2",
+        "entities": "^2.0.0"
+      }
+    },
     "http-errors": {
       "version": "1.7.2",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@@ -278,6 +375,15 @@
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
       "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
     },
+    "kairoscope-db-models": {
+      "version": "0.1.3",
+      "resolved": "https://registry.kairoscope.net/kairoscope-db-models/-/kairoscope-db-models-0.1.3.tgz",
+      "integrity": "sha512-zDfqr+mYdvbnfrRtZ8oM4Z+BGQqp99Dplf3A1Y+M+xxRUeIVLfjaGnfkAE/YSlBH1/kgGP7kHsEX90/HRcZCyQ==",
+      "requires": {
+        "chalk": "^4.1.2",
+        "mongoose": "^6.0.8"
+      }
+    },
     "kareem": {
       "version": "2.3.2",
       "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.3.2.tgz",
@@ -410,6 +516,14 @@
       "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.6.5.tgz",
       "integrity": "sha512-C/v856DBijUzHcHIgGpQoTrfsH3suKIRAGliIzCstatM2cAa+MYX3LuyCrABiO/cdJTxgBBHXxV1ztiqUwst5A=="
     },
+    "nth-check": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.0.1.tgz",
+      "integrity": "sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==",
+      "requires": {
+        "boolbase": "^1.0.0"
+      }
+    },
     "on-finished": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@@ -418,6 +532,19 @@
         "ee-first": "1.1.1"
       }
     },
+    "parse5": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
+      "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw=="
+    },
+    "parse5-htmlparser2-tree-adapter": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz",
+      "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==",
+      "requires": {
+        "parse5": "^6.0.1"
+      }
+    },
     "parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -575,6 +702,11 @@
         "punycode": "^2.1.1"
       }
     },
+    "tslib": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz",
+      "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw=="
+    },
     "type-is": {
       "version": "1.6.18",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@@ -584,6 +716,11 @@
         "mime-types": "~2.1.24"
       }
     },
+    "uniqid": {
+      "version": "5.4.0",
+      "resolved": "https://registry.npmjs.org/uniqid/-/uniqid-5.4.0.tgz",
+      "integrity": "sha512-38JRbJ4Fj94VmnC7G/J/5n5SC7Ab46OM5iNtSstB/ko3l1b5g7ALt4qzHFgGciFkyiRNtDXtLNb+VsxtMSE77A=="
+    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

package.json

@@ -19,8 +19,11 @@
   "license": "ISC",
   "dependencies": {
     "chalk": "^4.1.2",
+    "cheerio": "^1.0.0-rc.10",
     "express": "^4.17.1",
+    "kairoscope-db-models": "^0.1.3",
     "mongoose": "^6.0.8",
-    "nodemailer": "^6.6.5"
+    "nodemailer": "^6.6.5",
+    "uniqid": "^5.4.0"
   }
 }

readme.md

@@ -2,13 +2,17 @@
 
 An API service for sending email using the Amazon Web Services Simple Email Service (AWS SES). It connects to a MongoDB database for storing data.
 
+### Resources
+
+- [Juice](https://github.com/Automattic/juice) – Juice inlines CSS stylesheets into your HTML source.
+
 ### Tasks
 
 - [X] Send mail endpoint
 - [X] Send email thru AWS SES
 - [X] Save email data on DB
-- [ ] Append Pixel tracking
-- [ ] Change Links to add tracking
+- [X] Append Pixel tracking
+- [X] Change Links to add tracking