from https import Http
from parse import Parse
from setting import headers
from setting import cookies
import time
import logging
import codecs
 
#logging.basicConfig函数为日志系统做基本配置：Python写比较大型的程序，用日志系统会方便发现错误。
logging.basicConfig(level=logging.ERROR,
                    format='%(asctime)s Process%(process)d:%(thread)d %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    filename='diary.log',
                    filemode='a')
 
 
def getInfo(url, para): #输入网址、参数实现获取网页上的内容
    generalHttp = Http()  
    htmlCode = generalHttp.post(url, para=para, headers=headers, cookies=cookies) 
    generalParse = Parse(htmlCode) 
    pageCount = generalParse.parsePage()  
    info = []  
 
def getInfoDetail(generalParse): 
    info = generalParse.parseInfo()
    return info
 
 
def processInfo(info, para): #实现将爬取的网页内容存入到xls表格文件
    logging.error('Process start') 
    try: 
        title = '公司名称\t公司类型\t融资阶段\t标签\t公司规模\t公司所在地\t职位类型\t学历要求\t福利\t薪资\t工作经验\t发布时间\n'
        file = codecs.open('%s市有关区块链的职位信息表.xls' % para['city'], 'w', 'utf-8')
        file.write(title) #file.write函数先将xls表格的列名写入xls文件内
        for p in info: #for循环依次将十二个字段的内容写入xls文件
            line = str(p['companyName']) + '\t' + str(p['companyType']) + '\t' + str(p['companyStage']) + '\t' + \
                   str(p['companyLabel']) + '\t' + str(p['companySize']) + '\t' + str(p['companyDistrict']) + '\t' + \
                   str(p['positionType']) + '\t' + str(p['positionEducation']) + '\t' + str(
                p['positionAdvantage']) + '\t' + \
                   str(p['positionSalary']) + '\t' + str(p['positionWorkYear']) + '\t' + str(p['createTime'])+'\n'
            file.write(line) #最后将line写入xls文件
        file.close() #file.close()关闭xls文件
        return True
    except Exception as e:
        print(e)
        return None
 
 
def main(url, para): #函数开始进行爬取网址
    logging.error('Main start') 
    if url:
        info = getInfo(url, para)  
        flag = processInfo(info, para) 
        return flag
    else:
        return None
 
if __name__ == '__main__':
    kdList = [u'区块链']  
    cityList = [u'上海', u'北京']  
    url = 'https://www.lagou.com/jobs/positionAjax.json' 
    for city in cityList:
        print('现在开始爬取%s' % city)
        para = {'first': 'true', 'pn': '1', 'kd': kdList[0], 'city': city} #{，页数，城市}
        flag = main(url, para) 
        if flag: #if人性化输出是否爬取成功
            print('%s爬取成功！' % city)
        else:
            print('%s爬取失败！' % city)