python操作数据库之批量导入

python操作数据库之批量导入

---

批量从mongodb导入数据到redis

• 结构图如下

python->mongodb:python取出mongodb数据

mongodb->redis:mongodb数据经过处理后保存到redis

• python代码如下：

# -*- coding: utf-8 -*-

import redis

import pymongo

import logging

class savequestionmafengwo(object):

    def __init__(self):

        logging.basicConfig(filename = "./save-mafengwo-question.log", level = logging.DEBUG)

        self.redis_pool_conn=self.redis_pool()

        self.redis_db=self.connection_redis_db();

        self.mongo_db=self.connection_mongo_db();

        self.url='http://www.mafengwo.cn/wenda/detail-';

        self.hz='.html';

    def redis_pool(self):

        pool_redis=redis.ConnectionPool(host='IP',port=PORT,db=DB,password=PASS);

        return pool_redis;

    def connection_redis_db(self):

        pool=self.redis_pool_conn;

        redis_db = redis.Redis(connection_pool=pool)

        return redis_db;

    def get_data(self,start):

        mongo=self.mongo_db

        qas=mongo.find({"source":"mafengwo","type":"question"},{"qid":1}).skip(start).limit(40000);

        return qas;

    def put_data(self,start):

        qas=self.get_data(start);

        i=start

        for qa in qas:

            try:

                urlF = self.url + str(qa['qid']) + self.hz;

                self.redis_set_value(urlF);

                logging.debug(str(i) + "------" + urlF)

                i = i + 1

            except:

                logging.debug("error:" + str(qa['_id']))

                raise

    def connection_mongo_db(self):

        client = pymongo.MongoClient("IP", PORT)

        mydb = client.hello # new a database

        mydb.authenticate('USER', 'PASS') # check auth

        muser = mydb.QA# new a table

        return muser;

    def redis_set_value(self,urlF):

        db =self.redis_db

        db.set(urlF,"1");

if __name__ == '__main__':

    savequestionmafengwo=savequestionmafengwo();

    start={0,40000}

    for st in start:

        savequestionmafengwo.put_data(start)