forked from simoncos/weibo-complaint-crawler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
mongo.py
21 lines (16 loc) · 752 Bytes
/
mongo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
from pymongo import MongoClient
from conf import MONGO_HOST, MONGO_PORT, MONGO_DATABASE, MONGO_USERNAME, MONGO_PASSWORD, MONGO_COLLECTION
class MongoHelper:
def __init__(self):
self.client = MongoClient(MONGO_HOST, MONGO_PORT)
self.db = self.client[MONGO_DATABASE]
self.db.authenticate(MONGO_USERNAME, MONGO_PASSWORD)
self.collection = self.db[MONGO_COLLECTION]
def update(self, data):
if type(data) is list:
for i in data:
self.collection.replace_one({'url': i['url']}, i, upsert=True)
else:
self.collection.replace_one({'url': data['url']}, data, upsert=True)
def getCrawledUrls(self):
return [i['url'] for i in self.collection.find({})]