Merge pull request #12 from uupers/nodeapp

more error info -> app
uupers · Feb 24, 2018 · 744a0fe · 744a0fe
2 parents bb7cc35 + 05787c7
commit 744a0fe
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 23 deletions.
diff --git a/app/assets/mongo.js b/app/assets/mongo.js
@@ -0,0 +1,34 @@
+var msg;
+// get database stats
+var getDbStats = function(db, callback) {
+db.command({'dbStats': 1},
+function(err, results) {
+   console.log(results);
+   callback();
+   return results;
+}
+);
+};
+
+var MongoClient = require('mongodb').MongoClient
+  , assert = require('assert');
+
+
+
+// Connection URL
+var url = 'mongodb://spiderrd:[email protected]:37017/bilibili_spider';
+
+
+MongoClient.connect(url, function (err, client) {
+  assert.equal(null, err);
+
+  var db = client.db('bilibili_spider');
+
+    msg = getDbStats(db, function() {
+      client.close();
+     });
+}); 
+
+
+
+
diff --git a/app/assets/vue-app.js b/app/assets/vue-app.js
@@ -0,0 +1,39 @@
+var MongoClient = require('mongodb').MongoClient
+  , assert = require('assert');
+
+
+var vm = new Vue({
+el: '#data-html',
+data: {
+   url : 'mongodb://spiderrd:[email protected]:37017/bilibili_spider',
+   loaded: "loaded",
+   dataStats: ""
+   },
+   mounted: function () {
+      this.loaded = "loaded database";
+   },
+   methods: {
+      getDbStats: function(db, callback) {
+      db.command({'dbStats': 1},
+      function(err, results) {
+         console.log(results);
+         callback();
+         return results;
+      }
+      );
+      }
+   },
+   computed: {
+      dataStats: MongoClient.connect(url, function (err, client) {
+         assert.equal(null, err);
+
+         var db = client.db('bilibili_spider');
+
+            stats = getDbStats(db, function() {
+               client.close();
+            }).objects;
+
+            return stats
+         })
+   }
+})
diff --git a/app/bilicrawler.js b/app/bilicrawler.js
@@ -1,5 +1,10 @@
 const loghtml = document.getElementById('log-process')
 
+///////////////////
+// For Kernel
+///////////////////
+
+//version 20180225-1
 const superagent = require('superagent');
 var moment = require('moment');
 moment.locale('zh-cn');
@@ -74,13 +79,13 @@ const packageFetchInsertAsync = async (pid, mids) => {
             break // 结束本次任务，尝试下个任务
         }
     }
-    await sleep(12000)
+    await sleep(10000)
     if (cardList.length === midSize) {
         await uploadPackageAsync(pid, cardList)
         console.log(`${nowstr()} Send package ${pid}`);
     } else {
-        console.error(`${nowstr()} failed to fetch info, mids=${mids}`);
-        logit(`${nowstr()} failed to fetch info, mids=${mids}`); 
+        console.error(`${nowstr()} failed to fetch info，finished/all=${cardList.length}/${midSize}, mids=${mids}`);
+        logit(`${nowstr()} failed to fetch info，finished/all=${cardList.length}/${midSize}, mids=${mids}`); 
     }
 }
 
@@ -105,7 +110,9 @@ const run = async () => {
 // start code
 // run();
 
-
+///////////////////
+// For Electron
+///////////////////
 
 // Define function that export result to html tag
 function logit(elem) {

diff --git a/app/index.html b/app/index.html
@@ -6,14 +6,6 @@
     <title>BiliSpider</title>
     <link rel="stylesheet" href="assets/css/bulma.css">
     <link rel="stylesheet" href="assets/css/style.css">
-    <!-- <script>
-    var shell = require('electron').shell;
-    //open links externally by default
-    $(document).on('click', 'a[href^="http"]', function(event) {
-        event.preventDefault();
-        shell.openExternal(this.href);
-    });
-  </script> -->
   </head>
 
   <body>
@@ -42,27 +34,39 @@ <h1 class="title is-1">
             </div>
 
             <div class="content">
-
-            <div class="text-centered">
-                <button class = "button is-dark is-centered" id = "btn-run">贡献你的计算力</button>
+                  <div class="text-centered">
+                      <button class = "button is-dark is-centered" id = "btn-run">贡献你的计算力</button>
+                  </div>
             </div>
           </div>
-          </div>
         </section>
 
 <section>        
   <div id="log-pre">
       <span class="tag is-light">LOG:</span>
     <pre id="log-process">
 
-    </pre
+    </pre>
     </div>
-  </section>
+</section>
+
+<!-- <section>
+
+
+  <div>
+      <span class="tag is-light">Database:</span>
+    <div id="data-html">
+        {{ loaded }}
+    </div>
+    </div>
+</section> -->
 
 
 
-        <script src = "./bilicrawler.js" ></script>
         <script src = "./assets/vue/vue.min.js" ></script>
+        <script src = "./bilicrawler.js" ></script>
+        <!-- <script src = "./assets/mongo.js" ></script> -->
+        <!-- <script src = "./assets/vue-app.js" ></script> -->
     <script>
       // You can also require other files to run in this process
       require('./renderer.js')

diff --git a/app/package-lock.json b/app/package-lock.json
diff --git a/app/package.json b/app/package.json
@@ -1,6 +1,6 @@
 {
   "name": "bilispider",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Bilibili spider to crawl data",
   "main": "main.js",
   "scripts": {
@@ -21,8 +21,10 @@
     "electron-packager": "^11.0.1"
   },
   "dependencies": {
+    "assert": "^1.4.1",
     "bulma": "",
     "moment": "",
+    "mongodb": "^3.0.3",
     "superagent": ""
   },
   "build": {