【機械学習】WordPressでより高精度な関連記事を表示させたい【Python】

import pymysql

#投稿を取得していく

#mysql接続
db = pymysql.connect(user="ユーザー",passwd="パスワード",host="localhost",db="データベース名")
cur = db.cursor()

#投稿を取得するクエリ
query = "SELECT ID,post_content,post_title FROM wp_posts WHERE post_type = 'post'"
cur.execute(query)
post_list = cur.fetchall()
print(post_list)

from bs4 import BeautifulSoup

#それぞれの投稿について処理
for (post_id,post_content,post_title) in post_list:

    soup = BeautifulSoup(post_content, "html.parser")
    content_p_list = [p.get_text() for p in soup.find_all('p')]
    content_p_list.append(post_title)

import pymysql
from bs4 import BeautifulSoup
from sudachipy import tokenizer
from sudachipy import dictionary
import re
import math

#tokenizerの設定
tokenizer_obj = dictionary.Dictionary().create()
mode = tokenizer.Tokenizer.SplitMode.C

#mysql接続
db = pymysql.connect(user="ユーザー",passwd="パスワード",host="localhost",db="データベース名")
cur = db.cursor()

#投稿を取得するクエリ
query = "SELECT ID,post_content,post_title FROM wp_posts WHERE post_type = 'post'"
cur.execute(query)

post_list = cur.fetchall()

tfidf_dic = {}

#それぞれの投稿について処理
for (post_id,post_content,post_title) in post_list:

    soup = BeautifulSoup(post_content, "html.parser")
    content_p_list = [p.get_text() for p in soup.find_all('p')]
    content_p_list.append(post_title)

    tf_dic = {}
    word_sum = 0

    for each_paragraph in content_p_list:
        for each_token in tokenizer_obj.tokenize(each_paragraph,mode):

            kind = each_token.part_of_speech()[0]
            if re.search(r'^助動?詞$',kind):
                continue
            
            word = each_token.normalized_form()
            word = re.sub('[\n 　\t!"#$%&\'\\\\()*+,-./:;<=>?@[\\]^_`{|}~「」〔〕“”〈〉『』【】＆＊・（）＄＃＠。、？！｀＋￥％]','',word)

            if word in tf_dic:
                tf_dic[word] = tf_dic[word] + 1
                word_sum += 1
            elif word:
                tf_dic[word] = 1
                word_sum += 1
    
    for each_word,each_num in tf_dic.items():
        
        if each_word in tfidf_dic:
            tfidf_dic[each_word].update({post_id:each_num / word_sum})
        else:
            tfidf_dic[each_word] = {post_id:each_num / word_sum}             
            
post_sum = len(post_list)

#tfidfリストを完成させていく
for each_word in tfidf_dic:
    this_idf = math.log2(post_sum / (len(tfidf_dic[each_word]))) #idfの値

    for each_post in tfidf_dic[each_word]:
        this_tf = tfidf_dic[each_word][each_post]
        this_tfidf = this_tf * this_idf

        tfidf_dic[each_word][each_post] = this_tfidf

print(tfidf_dic)

#主成分分析
post_tfidf_df = pd.DataFrame(tfidf_dic).fillna(0)

pca = PCA(n_components=10)
new_tfidf = pca.fit_transform(post_tfidf_df.iloc[:,:].values)

sim = cosine_similarity(new_tfidf)

similarity_list = {}
for each_post in range(len(sim)):
    this_post_id = post_list[each_post]
    similarity_list[str(this_post_id)] = {}

    for each_target_post in range(len(sim[each_post])):
        if sim[each_post][each_target_post] > 0.9 and each_post != each_target_post:
            target_post_id = post_list[each_target_post]
            similarity_list[str(this_post_id)][str(target_post_id)] = sim[each_post][each_target_post]

//類似度行列を取得
$similarity_list = file_get_contents("～/post_similarity.json"); //類似度行列ファイル
$similarity_list = json_decode($similarity_list,true);

$this_similarity_list = array_key_exists($post_id,$similarity_list) ? $similarity_list[$post_id] : array();

arsort($this_similarity_list);
$max = 5;
$i = 0;

foreach($this_similarity_list as $each_post => $each_score){
    if($i >= $max){break;}
    $thumbnail_url = get_the_post_thumbnail_url($each_post,"medium");
    $title = get_the_title($each_post);
    $url = "https://bablishe.com/?p={$each_post}";
    
    $relative_DOM .= "<li><a href='{$url}'><img src='{$thumbnail_url}' alt='投稿のサムネイル画像' loading='lazy'></a>";
    $relative_DOM .= "<a class='post_title' href='{$url}'>{$title}</a></li>";
    $i++;
}

$random_post_id = array_rand($similarity_list,1); //ランダムな記事も

$thumbnail_url = get_the_post_thumbnail_url($random_post_id,"medium");
$title = get_the_title($random_post_id);
$url = "https://bablishe.com/?p={$random_post_id}";

$relative_DOM .= "<li><a href='{$url}'><img src='{$thumbnail_url}' alt='投稿のサムネイル画像' loading='lazy'></a>";
$relative_DOM .= "<a class='post_title' href='{$url}'>{$title}</a></li>";

$result_data_set = array("pp_DOM"=>$result_DOM,"rp_DOM"=>$relative_DOM);


echo json_encode($result_data_set);

done(function (data) {
    data = JSON.parse(data);
    jQuery("ul#sidebar_popular_list").html(data["pp_DOM"]);
    
  });