価格ドットコムをスクレイピングしてゲーミングPCを調査してみた(前編)(2022/10/14更新)

# ライブラリのインポート

import lxml.html
import matplotlib.pyplot as plt
import pandas as pd
import requests
import time

from bs4 import BeautifulSoup

url="https://kakaku.com/pc/gaming-pc/itemlist.aspx"
res = requests.get(url)
soup = BeautifulSoup(res.text,"lxml")
soup.find(class_="result").span.text

pages = int(soup.find(class_="result").span.text)//40 + 1
print(pages)

# 上で確認したページ数を入力
pages = 18

# データダウンロード
page_url_base = "https://kakaku.com/pc/gaming-pc/itemlist.aspx?pdf_vi=d&pdf_pg="

res = []
for i in range(1,pages+1):
    page_url = page_url_base + str(i)
    print(page_url)
    res.append(requests.get(page_url))
    time.sleep(5)

elems = soup.find_all(class_="tr-border")

len(elems)

i = 0
elems[3*i+2]

<tr class="tr-border">
    <td class="sel alignC ckbtn" rowspan="2"><input name="ChkProductID" onclick="setProductID('K0001446191','V077');"
            type="checkbox" value="K0001446191" /></td>
    <td class="end checkItem" colspan="15">
        <table>
            <tr>
                <td class="ckitemLink"><a class="ckitanker" href="https://kakaku.com/item/K0001446191/"><span>マウスコンピューター
                            <span class="dir">【直販モデル】</span>　</span>G-Tune PM-B-3060Ti-KK 価格.com限定 Core i5 12400F/RTX
                        3060Ti/16GBメモリ/512GB NVMe SSD搭載モデル #2205PM-B660W11-3060Ti-KK</a></td>
                <td class="addFavoriteBtn"><span class="showNum"
                        onclick="location.href='https://ssl.kakaku.com/auth/mypage/favorite/entry.aspx?PrdKey=K0001446191&lid=facet_favorite'; return false;"><span
                            class="txt">お気に入り登録</span><span class="num">52</span></span></td>
            </tr>
        </table>
    </td>
</tr>

elems[i*3+2].find('a').attrs['href']

elems[i*3+2].find('span').text.split()

elems[i*3+2].find('a').text.split('\u3000')[1]

length = (len(elems)-2)//3

df = pd.DataFrame()
df['maker'] = [elems[i*3+2].find('span').text.split()[0] for i in range(length)]
df['limited'] = ["" if len(elems[i*3+2].find('span').text.split())==1 else elems[i*3+2].find('span').text.split()[1] for i in range(length)]
df['product'] = [elems[i*3+2].find('a').text.split('\u3000')[1] for i in range(length)]

df['URL'] = [elems[i*3+2].find('a').attrs['href'] for i in range(length)]

df.shape

df.head()

<tr class="tr-border">
    <td class="alignC"><a class="withIcnLimited" href="https://kakaku.com/item/K0001446191/"><span
                class="withIcnLimitedBox"><span class="icnLimited">限定</span></span><img
                alt="G-Tune PM-B-3060Ti-KK 価格.com限定 Core i5 12400F/RTX 3060Ti/16GBメモリ/512GB NVMe SSD搭載モデル #2205PM-B660W11-3060Ti-KK"
                height="60"
                onerror="this.src='https://img1.kakaku.k-img.com/images/productimage/m/nowprinting.gif';this.onerror=null"
                onload="resizeImageFix(this,80,60)"
                src="https://img1.kakaku.k-img.com/images/productimage/m/K0001446191.jpg" width="80" /></a><span
            class="dFeatureIcn">オプション特典付</span></td>
    <td class="td-price">
        <ul>
            <li class="pryen"><a href="https://kakaku.com/item/K0001446191/">¥169,800</a></li>
            <li class="prshop">マウス<br />（全1店舗）</li>
        </ul>
    </td>
    <td class="swrank1" style="display:none;"><span class="withRankIcn fontBold">1位</span></td>
    <td class="swrank2"><span class="withRankIcn fontBold">1位</span></td>
    <td><span class="withStarIcn fontBold">5.00</span><br /><a
            href="https://review.kakaku.com/review/K0001446191/#tab">(6件)</a></td>
    <td><a href="https://bbs.kakaku.com/bbs/K0001446191/#tab">13件</a></td>
    <td class="swdate1">2022/5/30 </td>
    <td class="swdate2" style="display:none;">2022/5/30</td>
    <td>GeForce RTX 3060Ti</td>
    <td>第12世代 インテル Core i5 12400F(Alder Lake)</td>
    <td>19841</td>
    <td>16GB</td>
    <td>M.2 SSD：512GB</td>
    <td><span class="sortBox"><a href="/pc/gaming-pc/itemlist.aspx?pdf_Spec107=21&pdf_vi=d"
                onclick="FSNPopup.open(this, 0, 2);return false;">Windows 11 Home</a><span class="variBlnHide"
                id="vbcnavK0001446191_Spec107"></span></span></td>
    <td class="end"><span class="sortBox"><a href="/pc/gaming-pc/itemlist.aspx?pdf_Spec101=2&pdf_vi=d"
                onclick="FSNPopup.open(this, 0, 3);return false;">タワー</a><span class="variBlnHide"
                id="vbcnavK0001446191_Spec101"></span></span></td>
</tr>

i = 0
int(elems[i*3+3].find(class_="pryen").text.replace(',','')[1:])

elems[i*3+3].find(class_="prshop").text

# 注目ランク
elems[i*3+3].find(class_="swrank1").text

# 売上ランク
elems[i*3+3].find(class_="swrank2").text

# 登録日
"".join(elems[i*3+3].find(class_="swdate1").text.split())

# 発売日
"".join(elems[i*3+3].find(class_="swdate2").text.split())

# Graphics board
elems[i*3+3].find_all("td")[8].text

# CPU
elems[i*3+3].find_all("td")[9].text

# CPUスコア
int("".join(elems[i*3+3].find_all("td")[10].text.split()) or 0)

# メモリ容量
elems[i*3+3].find_all("td")[11].text

# ストレージ
elems[i*3+3].find_all("td")[12].text

# OS
elems[i*3+3].find_all("td")[13].text

# 筐体
elems[i*3+3].find_all("td")[14].text

<tr class="tr-border">
    <td class="end ckitemSpec" colspan="16">
        <div class="ckitemSpecInnr"><span class="ckitemSpecInnrTtl">【スペック】</span><br /><span
                class="specType">ビデオメモリ：</span><span class="sortBox"><span class="typeClk"
                    onclick="FSNPopup.open(this, 0, 0);return false;">8GB</span><span class="variBlnHide"
                    id="vbcnavK0001446191_Spec317"></span></span>　<span class="specType">コア数：</span><span
                class="sortBox"><a href="/pc/gaming-pc/itemlist.aspx?pdf_Spec310=6&pdf_vi=d"
                    onclick="FSNPopup.open(this, 0, 1);return false;">6コア</a><span class="variBlnHide"
                    id="vbcnavK0001446191_Spec310"></span></span>　<span class="specType">CPU周波数：</span>2.5GHz　<span
                class="specType">二次キャッシュ容量：</span>7.5MB　<span class="specType">三次キャッシュ容量：</span>18MB　<span
                class="specType">メモリ最大容量：</span>32GB　<span class="specType">メモリ種類：</span>DDR4 PC4-25600　<span
                class="specType">全メモリスロット数：</span>2　<span class="specType">空メモリスロット数：</span>0　<span
                class="specType">OS：</span>Windows 11 Home 64bit　<span class="specType">その他機能：</span>8K出力　<span
                class="specType">インターフェース：</span>HDMI端子、DisplayPort、USB3.1 Gen1(USB3.0)、USB3.2 Gen2x2 Type-C　<span
                class="specType">LAN：</span>10/100/1000Mbps　<span
                class="specType">サイズ：</span>[本体]約189×396×390mm、[突起物含む]約194×411×400mm　<span
                class="specType">重量：</span>約10.3kg　<span class="specType">カラー：</span>ブラック系　</div>
    </td>
</tr>

i = 0
elems[i*3+4].text

i = 0
elems[i*3+4].text.replace("\n","").replace("【スペック】","").split("\u3000")[:-1]

# DataFrameの初期化
gamingPC = pd.DataFrame()

# DataFrameの要素を取得し、つなぎ合わせる
for page in range(len(res)):
    soup = BeautifulSoup(res[page].text,"lxml")
    elems = soup.find_all(class_="tr-border")
    length = (len(elems)-2)//3

    df = pd.DataFrame()
    df['maker'] = [elems[i*3+2].find('span').text.split()[0] for i in range(length)]
    df['limited'] = ["" if len(elems[i*3+2].find('span').text.split())==1 else elems[i*3+2].find('span').text.split()[1] for i in range(length)]
    df['product'] = [elems[i*3+2].find('a').text.split('\u3000')[1] for i in range(length)]
    df['price'] = [int(elems[i*3+3].find(class_="pryen").text.replace(',','')[1:]) for i in range(length)]
    df['shop'] = [elems[i*3+3].find(class_="prshop").text for i in range(length)]
    df['at_rank'] = [int(elems[i*3+3].find(class_="swrank1").text.replace('位','').replace('-','0')) for i in range(length)]
    df['sales_rank'] = [int(elems[i*3+3].find(class_="swrank2").text.replace('位','').replace('-','0')) for i in range(length)]
    df['reg_date'] = ["".join(elems[i*3+3].find(class_="swdate1").text.split()) for i in range(length)]
    df['release_date'] = ["".join(elems[i*3+3].find(class_="swdate2").text.split()) for i in range(length)]
    df['graphics_board'] = [elems[i*3+3].find_all("td")[8].text for i in range(length)]
    df['CPU'] = [elems[i*3+3].find_all("td")[9].text for i in range(length)]
    df['CPU_score'] = [int("".join(elems[i*3+3].find_all("td")[10].text.split()) or 0) for i in range(length)]
    df['memory'] = [elems[i*3+3].find_all("td")[11].text for i in range(length)]
    df['storage'] = [elems[i*3+3].find_all("td")[12].text for i in range(length)]
    df['OS'] = [elems[i*3+3].find_all("td")[13].text for i in range(length)]
    df['body'] = [elems[i*3+3].find_all("td")[14].text for i in range(length)]
    df['spec'] = [elems[i*3+4].text.replace("\n","").replace("【スペック】","").split("\u3000")[:-1] for i in range(length)]
    df['product_url'] =  [elems[i*3+2].find('a').attrs['href'] for i in range(length)]
    
    gamingPC = pd.concat([gamingPC, df], axis=0)

# インデックスの再設定
gamingPC = gamingPC.reset_index(drop=True)

gamingPC

# pickle形式でDataFrameのデータを保存
gamingPC.to_pickle("gamingPC_data202208_1.pickle")

import pandas as pd
gamingPC = pd.read_pickle("gamingPC_data202208_1.pickle")