import urllib2
import bs4

url = 'http://finance.daum.net/quote/all.daum?type=S&stype=P'
html = urllib2.urlopen(url).read()

len(html)

print html[:1000]

soup = bs4.BeautifulSoup(html)

title = soup.find('title')
print title
print title.text

print len(soup.find_all('td'))
print len(soup.find_all('td','txt'))

soup = bs4.BeautifulSoup(html, 'html.parser')
print len(soup.find_all('td'))
print len(soup.find_all('td','txt'))

for i in soup.find_all('td','txt')[:20]:
    print i.text

for i in soup.find_all('td','txt')[:20]:
    print i.text, i.a['href']

company_dic = {}
for i in soup.find_all('td','txt'):
    company_dic[i.text] = i.a['href'].split('=')[1]

print len(company_dic)
company_dic[u'AK홀딩스']

def get_company_list(url):
    soup = bs4.BeautifulSoup(urllib2.urlopen(url).read(),'html.parser')
    company_dic = {}
    for i in soup.find_all('td','txt'):
        company_dic[i.text] = i.a['href'].split('=')[1]
    return company_dic

kospi_url = 'http://finance.daum.net/quote/all.daum?type=U&stype=P'
kosdaq_url = 'http://finance.daum.net/quote/all.daum?type=U&stype=Q'

kosdaq_dic = get_company_list(kosdaq_url)
for k, v in kosdaq_dic.items()[:20]:
    print k, v

import pymysql # pip install pymysql
mysql_host = "127.0.0.1"
mysql_user = "root"
mysql_passwd = "mysql"
mysql_db = "stock"
mysql_port = 3306
conn = pymysql.connect(host=mysql_host, user=mysql_user, passwd=mysql_passwd, db=mysql_db, port=mysql_port, charset='utf8')
conn.autocommit(True)
cur = conn.cursor()

'''
CREATE TABLE `COMPANY2` (
  `company_id` varchar(11) NOT NULL DEFAULT '',
  `company_name` varchar(255) DEFAULT '',
  `company_type` tinyint(10) DEFAULT NULL,
  PRIMARY KEY (`company_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
'''

for k,v in kosdaq_dic.items():
    cur.execute("replace into COMPANY2 (company_id, company_name, company_type ) values ('{}', '{}', '{}')".format( v.encode('utf-8'), k.encode('utf-8'), 0 ))

for k,v in company_dic.items():
    cur.execute("replace into COMPANY2 (company_id, company_name, company_type ) values ('{}', '{}', '{}')".format( v.encode('utf-8'), k.encode('utf-8'), 1 ))

cur.execute("""select * from COMPANY2""")
cur_tmp = cur.fetchall()
for i in cur_tmp[:10]:
    print i[0], i[1], i[2]

import requests
import urllib
import bs4
import json
import re
import crawlib
import datetime
import sys
import crawlib
import pymysql # pip install pymysql

mysql_host = "127.0.0.1"
mysql_user = "root"
mysql_passwd = "mysql"
mysql_db = "stock"
mysql_port = 3306
conn = pymysql.connect(host=mysql_host, user=mysql_user, passwd=mysql_passwd, db=mysql_db, port=mysql_port, charset='utf8')
conn.autocommit(True)
cur = conn.cursor()

def extract_table(html):
    soup = bs4.BeautifulSoup(html)
    th = soup.find_all("th","title")
    vertical_columns = [i.text.strip() for i in th]

    bg = soup.find_all("th",re.compile(r"^r"))[2:]
    horizon_columns = [crawlib.getOnlyDigit(i.text.strip()) for i in bg]

    cBk = [int(crawlib.getOnlyDigit(i.text)) for i in soup.find_all("td","num")]

    # colums_count_check
    if len(cBk) != (len(vertical_columns) * len(horizon_columns)):
        print(error)
        sys.exit()
        
    return vertical_columns, horizon_columns, cBk


def request_and_store(company_code, freq_typ, fin_typ=0):
    company_code = str(company_code).zfill(6)
    # zero padding 6
    url = 'http://companyinfo.stock.naver.com/v1/company/ajax/cF1001.aspx?cmp_cd={}&fin_typ={}&freq_typ={}'.format(company_code, str(fin_typ), freq_typ)
    print url
    headers = {
        'Accept-Language' : 'ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4',
        'Accept-Encoding' : 'gzip, deflate, sdch',
        'X-Requested-With' : 'XMLHttpRequest',
        'Connection' : 'keep-alive',
        'Accept' : 'text/html, */*; q=0.01',
        'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36',
        'Host' : 'companyinfo.stock.naver.com',
    }
    cookies = {
    }
    data_dic = {
    #     'freq_typ' : '''Y''',
    #     'fin_typ' : '''0''',
    #     'cmp_cd' : '''051910''',
    }
    data = ""
    for k, v in data_dic.items():
        data += urllib.quote_plus(k)+"="+urllib.quote_plus(v)+"&"
    res = requests.post(url, headers=headers, cookies=cookies, data=data)
    vertical_columns, horizon_columns, cBk = extract_table(res.content)

    cnt = 0
    print len(cBk)
    for value in cBk:
        vertical_i = cnt / len(horizon_columns)
        horizon_i = cnt % len(horizon_columns)
#         print vertical_columns[vertical_i], horizon_columns[horizon_i]
#         print value
        try:
            cur.execute("""replace into META (company_id, meta_type, meta_freq_type, meta_value, meta_date, meta_crawled_date ) values ('{}', '{}', '{}', '{}', '{}', '{}')""".format( 
                    company_code, vertical_columns[vertical_i].encode("utf-8"), freq_typ , value, datetime.datetime.strptime(horizon_columns[horizon_i],"%Y%m"), datetime.datetime.now())
                        )
        except Exception as e:
            print e
        cnt += 1
    print cnt

cur.execute("""select * from COMPANY2""")
cur_tmp = cur.fetchall()
for i in cur_tmp[:3]:
    request_and_store(i[0],'Q')
    request_and_store(i[0],'Y')

import logging
import pymongo
import crawlib
import time
from datetime import datetime
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool

collection = pymongo.MongoClient().stock.naver_discussion

class NaverCrawling(crawlib.Crawling):
    def get_item_list(self, cate, page):
        url = self.make_target_url(cate, page)
        soup = crawlib.getSoup(url)
        link_list = []
        for i in soup.find_all('tr',{'onmouseover':'mouseOver(this)'}):
            link = i.find_all('td')[1].a['href'].split('&st=&sw=&page')[0]
            link = crawlib.urljoin('http://finance.naver.com',link)

            # already exist this data
            if self.data_dic.has_key(link):
                print "exist", link
                continue

            self.data_dic[link] = {}
            self.data_dic[link]['date'] = i.find_all('td')[0].text.strip()
            self.data_dic[link]['title'] = i.find_all('td')[1].a['title'].strip()
            self.data_dic[link]['opinion'] = i.find_all('td')[2].text.strip()
            self.data_dic[link]['view'] = i.find_all('td')[4].text.strip()
            self.data_dic[link]['sympathy'] = i.find_all('td')[5].text.strip()
            self.data_dic[link]['unsympathy'] = i.find_all('td')[6].text.strip()
            self.data_dic[link]['_id'] = link

            link_list.append(link)
            # self.get_item(link)

        if len(link_list) == 0:
            return 'stop'

        pool = Pool(len(link_list))
        res = pool.map(self.get_item, link_list)

    def get_item(self, url, key=''):
        if not key:
            key = url
        soup = crawlib.getSoup(url)
        content = '\n'.join([i.text for i in soup.find_all('div','view_se')]).strip()
        user = soup.find('th','info').find('span').text.strip()
        nid = crawlib.get_param_from_url(url, 'nid')
        code = crawlib.get_param_from_url(url, 'code')

        self.data_dic[key]['content'] = content
        self.data_dic[key]['nid'] = nid
        self.data_dic[key]['code'] = code
        self.data_dic[key]['user'] = user

        print key, self.data_dic[key]['opinion']
        # runtime_check = time.time()
        collection.replace_one({"_id": key}, self.data_dic[key], upsert=True)
        # print time.time() - runtime_check


    # todo: mongo db library 화
    # todo: 최신글부터 가져오되 몇개이상(한페이지이상)  중복 감지시 stop

target_name = 'naver_stock_discussion'
naver_dicussion = NaverCrawling(target_name=target_name, base_url='http://finance.naver.com/item/board.nhn?', cate_param='code',
                                page_param='page', cate_list_path='category.txt', page_list=range(1, 1000))


naver_dicussion.crawling_cate_list(cate_delay=0, page_delay=0)
# 결과는 console 에서 확인 

import crawlib # https://github.com/dongsam/crawlib

res = crawlib.har2requests('finance.naver.com.har') # 각자 크롬 개발자도구통해 받은 har 파일 경로 입력 

# print res # 아래 코드 생성

import requests
import urllib
import json
from pprint import pprint
url = 'http://polling.finance.naver.com/api/realtime.nhn?query=SERVICE_ITEM:035420|SERVICE_RECENT_ITEM:035420&_callback='
headers = {
    'Accept-Language' : 'ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4',
    'Accept-Encoding' : 'gzip, deflate, sdch',
    'Connection' : 'keep-alive',
    'Accept' : '*/*',
    'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36',
    'Host' : 'polling.finance.naver.com',
    'Referer' : 'http://finance.naver.com/item/main.nhn?code=035420',
}
cookies = {
}
data_dic = {
}
data = ""
for k, v in data_dic.items():
    data += urllib.quote_plus(k)+"="+urllib.quote_plus(v)+"&"
res = requests.post(url, headers=headers, cookies=cookies, data=data)
res_str = res.content.decode('euc-kr').encode('utf-8')
res_json = json.loads(res_str)
pprint(res_json)