Data Driven Knowledgebase

!pip install streamlit

%%writefile app.py

import streamlit as st

st.title('口コミ分析')



from bs4 import BeautifulSoup

import urllib

import pandas as pd

...

!streamlit run app.py & sleep 3 && npx localtunnel --port 8501

%%writefile app.py

import streamlit as st

st.title('口コミ分析')



from bs4 import BeautifulSoup

import urllib

import pandas as pd

import requests

import re

import japanize_matplotlib

import matplotlib.pyplot as plt

import plotly.express as px

from janome.tokenizer import Tokenizer

from collections import Counter

from textblob import TextBlob



url = st.text_input('口コミ分析対象URL', placeholder='口コミ分析対象URL',value = 'https://www.jalan.net/yad382598/kuchikomi/2.HTML?screenId=UWW3701&idx=30&smlCd=136802&dateUndecided=1&yadNo=382598&distCd=01')



def get_data(url):

html = requests.get(url)

soup = BeautifulSoup(html.content,'html.parser')



url_list = [url]



for row in url_list:

row = str(row)

html = urllib.request.urlopen(row)

soup = BeautifulSoup(html, 'html.parser')



# 各要素を抽出

post_body = soup.find_all("p", class_='jlnpc-kuchikomiCassette__postBody')

lead = soup.find_all("p", class_='jlnpc-kuchikomiCassette__lead')

plan_info = soup.find_all("dl", class_='jlnpc-kuchikomiCassette__planInfoList')

post_date = soup.find_all("p", class_='jlnpc-kuchikomiCassette__postDate')

rate_list = soup.find_all("dl", class_='jlnpc-kuchikomiCassette__rateList')

hotel_name = soup.find_all("p", class_='jlnpc-styleguide-scope jlnpc-yado__subTitle')

rating = soup.find_all('span', class_='jlnpc-kuchikomi__point')

cat_table = soup.find_all("table", class_='jlnpc-kuchikomi__catTable')

c_label=soup.find_all("span", class_='c-label')



# テキストを抽出し、リストに格納

post_body_text = [element.get_text().strip() for element in post_body]

lead_text = [element.get_text().strip() for element in lead]

plan_info_text = [element.get_text().strip() for element in plan_info]

post_date_text = [element.get_text().strip() for element in post_date]

rate_list_text = [element.get_text().strip() for element in rate_list]

hotel_name_text = [element.get_text().strip() for element in hotel_name]

integrated_rate_text = [element.get_text().strip() for element in rating]

cat_table_text = [element.get_text().strip() for element in cat_table]

c_label_text = [element.get_text().strip() for element in c_label]



return post_body_text, lead_text, plan_info_text, post_date_text, rate_list_text, hotel_name_text, integrated_rate_text, cat_table_text, c_label_text



def create_data(post_body_text, lead_text, plan_info_text, post_date_text, rate_list_text, hotel_name_text, integrated_rate_text, cat_table_text, c_label_text):



sex = []

age = []

purpose = [] # 利用用途

room_type = [] # 部屋タイプ

meal_type = [] # 食事タイプ

post_date_text_v2 = [] #投稿日

periods = [] # 時期

plans = [] # プラン名

prices = [] # 価格帯

low_prices = [] # 価格帯

high_prices = [] # 価格帯

room_ratings = []

bath_ratings = []

breakfast_ratings = []

dinner_ratings = []

service_ratings = []

cleanliness_ratings = []

overall_room_rating = []

overall_bath_rating = []

overall_breakfast_rating = []

overall_dinner_rating = []

overall_service_rating = []

overall_cleanliness_rating = []



for i in range(0, len(c_label_text), 4): # 4つの要素ごとにループ

# 性別と年齢の抽出

sex_age = c_label_text[i].strip().split('/')

if len(sex_age) == 2:

sex.append(sex_age[0])

age.append(sex_age[1])

else:

sex.append(None)

age.append(None)



# 利用用途の抽出

purpose.append(c_label_text[i + 1].strip() if i + 1 < len(c_label_text) else None)



# 部屋タイプの抽出

room_type.append(c_label_text[i + 2].strip() if i + 2 < len(c_label_text) else None)



# 食事タイプの抽出

meal_type.append(c_label_text[i + 3].strip() if i + 3 < len(c_label_text) else None)



# 正規表現パターンの定義

date_pattern = r'\d{4}/\d{1,2}/\d{1,2}'



for text in post_date_text:

# 正規表現を使用して日付を抽出

match = re.search(date_pattern, text)

if match:

# 抽出した日付を新しいリストに追加

post_date_text_v2.append(match.group())



for item in plan_info_text:

if item.startswith('時期'):

periods.append(item.split('\n')[1].split('宿泊')[0]) # 時期の情報を追加

elif item.startswith('プラン'):

plans.append(item.split('\n')[1]) # プラン名の情報を追加

elif item.startswith('価格帯'):

prices.append(item.split('\n')[-1].split('円')[0]) # 価格帯の情報を追加

if '〜' in item.split('\n')[-1].split('円')[0]:

low_prices.append(item.split('\n')[-1].split('〜')[0])

high_prices.append(item.split('\n')[-1].split('円')[0].split('〜')[1])

elif '～' in item.split('\n')[-1].split('円')[0]:

low_prices.append(item.split('\n')[-1].split('～')[0])

high_prices.append(item.split('\n')[-1].split('円')[0].split('～')[1])



for item in rate_list_text:

lines = item.split('\n') # 各行に分割

for i in range(0, len(lines), 2): # 2つの要素ごとにループ（カテゴリー名と評価）

category = lines[i]

rating = lines[i + 1]

if '部屋' in category:

room_ratings.append(rating)

elif '風呂' in category:

bath_ratings.append(rating)

elif '料理(朝食)' in category:

breakfast_ratings.append(rating)

elif '料理(夕食)' in category:

dinner_ratings.append(rating)

elif '接客・サービス' in category:

service_ratings.append(rating)

elif '清潔感' in category:

cleanliness_ratings.append(rating)



for item in cat_table_text:

lines = item.split('\n')

# 各カテゴリの評価を抽出

overall_room_rating = lines[lines.index('部屋') + 1]

overall_bath_rating = lines[lines.index('風呂') + 1]

overall_breakfast_rating = lines[lines.index('料理（朝食）') + 1]

overall_dinner_rating = lines[lines.index('料理（夕食）') + 1]

overall_service_rating = lines[lines.index('接客・サービス') + 1]

overall_cleanliness_rating = lines[lines.index('清潔感') + 1]



# DataFrameを作成

data = pd.DataFrame({

'sex': sex,

'age': age,

'purpose': purpose,

'room_type': room_type,

'meal_type': meal_type,

'post_body': post_body_text,

'post_lead': lead_text,

'post_date': post_date_text_v2,

'periods': periods,

'plans' : plans,

'prices': prices,

'low_prices' : low_prices,

'high_prices' : high_prices,

'room_ratings' : room_ratings,

'bath_ratings' : bath_ratings,

'breakfast_ratings' : breakfast_ratings,

'dinner_ratings' : dinner_ratings,

'service_ratings' : service_ratings,

'cleanliness_ratings' : cleanliness_ratings

})





# ホテル名だけを抽出

hotel_names = [name.split('のクチコミ・評価')[0] for name in hotel_name_text]



# データセットのすべての行に同じホテル名を適用

data['ホテル名'] = hotel_names*len(data)

data['総合評価'] = integrated_rate_text*len(data)

# データセットに各カテゴリの評価の列を追加

data['総合部屋評価'] = overall_room_rating

data['総合風呂評価'] = overall_bath_rating

data['総合朝食評価'] = overall_breakfast_rating

data['総合夕食評価'] = overall_dinner_rating

data['総合接客・サービス評価'] = overall_service_rating

data['総合清潔感評価'] = overall_cleanliness_rating



# 評価データが文字列の場合、数値型に変換

data['room_ratings'] = pd.to_numeric(data['room_ratings'], errors='coerce')

data['bath_ratings'] = pd.to_numeric(data['bath_ratings'], errors='coerce')

data['breakfast_ratings'] = pd.to_numeric(data['breakfast_ratings'], errors='coerce')

data['dinner_ratings'] = pd.to_numeric(data['dinner_ratings'], errors='coerce')

data['service_ratings'] = pd.to_numeric(data['service_ratings'], errors='coerce')

data['cleanliness_ratings'] = pd.to_numeric(data['cleanliness_ratings'], errors='coerce')



return data



def show_data(data):

# データフレームを表示

st.write("Here's our data:")

st.write(data)





def plot_all_graphs(data):

# 'prices' ごとの性別割合with st.expander("価格帯ごとの性別割合"):

prices_sex_distribution = data.groupby('prices')['sex'].value_counts(normalize=True).unstack().fillna(0)

plt.figure(figsize=(10, 6))

prices_sex_distribution.plot(kind='barh', stacked=True)

plt.title('価格帯ごとの性別割合')

plt.xlabel('割合')

plt.ylabel('価格帯')

st.pyplot(plt)



# 性別の割合を計算して円グラフで表示

sex_counts = data['sex'].value_counts()

fig = px.pie(sex_counts, values='sex', names=sex_counts.index, title='性別の割合')

fig.update_layout(plot_bgcolor='black', paper_bgcolor='black', font_color='white')

st.plotly_chart(fig)



# 年齢の分布を計算して棒グラフで表示

age_counts = data['age'].value_counts()

fig = px.bar(age_counts, x=age_counts.index, y='age', title='年齢の分布')

fig.update_layout(plot_bgcolor='black', paper_bgcolor='black', font_color='white')

st.plotly_chart(fig)



def word_fec():



# Janomeの形態素解析器の初期化

tokenizer = Tokenizer()



# ストップワードの設定（独自に定義する必要があります）

stop_words = {'これ', 'は', 'です', 'が', '。', '、', 'し', 'の', '・'}



# ワードの頻出分析

words = []

for post in data['post_body']:

tokens = tokenizer.tokenize(post)

words.extend([token.surface for token in tokens if token.surface not in stop_words and token.part_of_speech.split(',')[0] in ['名詞','動詞', '形容詞']])



word_counts = Counter(words)



# 上位20単語を取得

top_words = word_counts.most_common(20)



# 単語とそのカウントを別々のリストに分ける

words, counts = zip(*top_words)



# 棒グラフの作成

fig, ax = plt.subplots()

ax.bar(words, counts)

ax.set_xlabel('単語')

ax.set_ylabel('頻度')

ax.set_title('上位20位の頻出単語')

ax.tick_params(axis='x', rotation=90)

st.pyplot(fig)



def pozi_naga():





# ポジティブ・ネガティブ意見の分析

data['sentiment'] = data['post_body'].apply(lambda x: TextBlob(x).sentiment.polarity)

data['opinion'] = data['sentiment'].apply(lambda x: 'positive' if x > 0 else ('negative' if x < 0 else 'neutral'))



# ポジティブ・ネガティブ意見のカウント

opinion_counts = data['opinion'].value_counts()



# 'post_date'が文字列型の場合、日付型に変換

data['post_date'] = pd.to_datetime(data['post_date'])



# 日付ごとにグループ化し、意見のカウントを取得

sentiment_counts_by_date = data.groupby('post_date')['opinion'].value_counts().unstack().fillna(0)



# グラフで推移を表示

fig, ax = plt.subplots()

sentiment_counts_by_date.plot(kind='line', ax=ax)

ax.set_title('口コミ属性別投稿数推移')

ax.set_xlabel('日付')

ax.set_ylabel('口コミ数')

ax.grid(True)

st.pyplot(fig)





# 「実行」ボタン

if st.button('実行'):

if url:

# get_data 関数を呼び出し、戻り値を複数の変数に代入

post_body_text, lead_text, plan_info_text, post_date_text, rate_list_text, hotel_name_text, integrated_rate_text, cat_table_text, c_label_text = get_data(url)

data = create_data(post_body_text, lead_text, plan_info_text, post_date_text, rate_list_text, hotel_name_text, integrated_rate_text, cat_table_text, c_label_text)

# データフレームを表示

show_data(data)

#基礎情報グラフを表示

plot_all_graphs(data)

word_fec()

pozi_naga()



else:

st.write('URLを入力してください。')