迁移web_img_manager代码到web_img文件
This commit is contained in:
vincent
parent
510b4bf0d4
commit
cfb8c9e078
4
main.py
4
main.py
@ -1,7 +1,7 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from requester import Requester
|
||||
from web_parser import Parser
|
||||
from web_img_manager import ImgManager
|
||||
from web_img import ImgManager
|
||||
|
||||
|
||||
def pre_batch_task(lines: list[str]):
|
||||
@ -17,7 +17,7 @@ def pre_batch_task(lines: list[str]):
|
||||
img_url_list = Parser(html_content).get_img_url_list()
|
||||
|
||||
img_manager = ImgManager(img_url_list, task_name)
|
||||
img_manager.batch_fetch_images()
|
||||
img_manager.batch_fill_image_data()
|
||||
img_manager.save_long_image()
|
||||
print(f"{task_name}, 完成!!")
|
||||
|
||||
|
||||
18
requester.py
18
requester.py
@ -1,10 +1,5 @@
|
||||
import os
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from PIL import Image
|
||||
import io
|
||||
from web_img import WebImg
|
||||
|
||||
|
||||
class Requester:
|
||||
@ -63,16 +58,3 @@ class Requester:
|
||||
else:
|
||||
print("Failed to download image after multiple retries, skipping.")
|
||||
return None
|
||||
|
||||
def fetch_images_to_img_obj(self, web_img: WebImg):
|
||||
url = web_img.url
|
||||
data = self.fetch_image(url)
|
||||
if data is None:
|
||||
task_name = web_img.task_name
|
||||
print(f"{task_name}, 下载图片失败")
|
||||
raise Exception(f"{task_name}, 下载图片失败")
|
||||
web_img.data = data
|
||||
|
||||
def batch_fetch_images_to_img_obj_list(self, web_img_list: list[WebImg]):
|
||||
with ThreadPoolExecutor() as executor:
|
||||
executor.map(self.fetch_images_to_img_obj, web_img_list)
|
||||
|
||||
77
web_img.py
77
web_img.py
@ -1,5 +1,76 @@
|
||||
from PIL import Image
|
||||
import io
|
||||
from requester import Requester
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
class WebImg:
|
||||
def __init__(self, file_name, url):
|
||||
self.task_name = file_name
|
||||
def __init__(self, task_name: str, url: str):
|
||||
self.task_name = task_name
|
||||
self.url = url
|
||||
self.data = None
|
||||
self.data = bytearray()
|
||||
|
||||
def fill_img_data(self):
|
||||
requester = Requester()
|
||||
url = self.url
|
||||
data = requester.fetch_image(url)
|
||||
if data is None:
|
||||
print(f"{self.task_name}, 下载图片失败")
|
||||
raise Exception(f"{self.task_name}, 下载图片失败")
|
||||
self.data = data
|
||||
|
||||
|
||||
class ImgManager:
|
||||
def __init__(self, img_url_list: list[str], task_name: str):
|
||||
self.img_url_list = img_url_list
|
||||
self.task_name = task_name
|
||||
self.img_list = self.__create_web_img_list()
|
||||
|
||||
def __create_web_img_list(self):
|
||||
img_list = []
|
||||
for url in self.img_url_list:
|
||||
img = WebImg(self.task_name, url)
|
||||
img_list.append(img)
|
||||
return img_list
|
||||
|
||||
def batch_fill_image_data(self):
|
||||
with ThreadPoolExecutor() as executor:
|
||||
executor.map(lambda web_img: web_img.fill_img_data(), self.img_list)
|
||||
|
||||
def concatenate_images_vertically(self):
|
||||
"""
|
||||
垂直拼接长图片
|
||||
"""
|
||||
try:
|
||||
# 计算拼接后的长图宽度和总高度
|
||||
max_width = max(
|
||||
Image.open(io.BytesIO(web_img.data)).width for web_img in self.img_list
|
||||
)
|
||||
total_height = sum(
|
||||
Image.open(io.BytesIO(web_img.data)).height for web_img in self.img_list
|
||||
)
|
||||
|
||||
# 创建一张新的长图
|
||||
long_image = Image.new(
|
||||
"RGB", (max_width, total_height), color=(255, 255, 255)
|
||||
)
|
||||
|
||||
# 依次将图片在垂直方向上拼接起来
|
||||
y_offset = 0
|
||||
for web_img in self.img_list:
|
||||
img = Image.open(io.BytesIO(web_img.data))
|
||||
img_width, img_height = img.size
|
||||
x_offset = (max_width - img_width) // 2 # 居中拼接
|
||||
long_image.paste(img, (x_offset, y_offset))
|
||||
y_offset += img_height
|
||||
|
||||
return long_image
|
||||
|
||||
except Exception as e:
|
||||
task_name = self.img_list[0].task_name
|
||||
print(f"{task_name}, 拼接图片失败：{e}")
|
||||
return None
|
||||
|
||||
def save_long_image(self):
|
||||
long_image = self.concatenate_images_vertically() # 垂直拼接长图片
|
||||
long_image.save(f"output/{self.img_list[0].task_name}.png") # 保存长图到本地
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
from requester import Requester
|
||||
from web_img import WebImg
|
||||
|
||||
from PIL import Image
|
||||
import io
|
||||
from web_img import WebImg
|
||||
from requester import Requester
|
||||
|
||||
|
||||
class ImgManager:
|
||||
def __init__(self, img_url_list: list[str], task_name):
|
||||
self.img_url_list = img_url_list
|
||||
self.task_name = task_name
|
||||
self.img_list = self.__create_web_img_list()
|
||||
|
||||
def __create_web_img_list(self):
|
||||
img_list = []
|
||||
for url in self.img_url_list:
|
||||
img = WebImg(self.task_name, url)
|
||||
img_list.append(img)
|
||||
return img_list
|
||||
|
||||
def batch_fetch_images(self):
|
||||
requester = Requester()
|
||||
requester.batch_fetch_images_to_img_obj_list(self.img_list)
|
||||
|
||||
def concatenate_images_vertically(self):
|
||||
"""
|
||||
垂直拼接长图片
|
||||
"""
|
||||
try:
|
||||
# 计算拼接后的长图宽度和总高度
|
||||
max_width = max(
|
||||
Image.open(io.BytesIO(web_img.data)).width for web_img in self.img_list
|
||||
)
|
||||
total_height = sum(
|
||||
Image.open(io.BytesIO(web_img.data)).height for web_img in self.img_list
|
||||
)
|
||||
|
||||
# 创建一张新的长图
|
||||
long_image = Image.new(
|
||||
"RGB", (max_width, total_height), color=(255, 255, 255)
|
||||
)
|
||||
|
||||
# 依次将图片在垂直方向上拼接起来
|
||||
y_offset = 0
|
||||
for web_img in self.img_list:
|
||||
img = Image.open(io.BytesIO(web_img.data))
|
||||
img_width, img_height = img.size
|
||||
x_offset = (max_width - img_width) // 2 # 居中拼接
|
||||
long_image.paste(img, (x_offset, y_offset))
|
||||
y_offset += img_height
|
||||
|
||||
return long_image
|
||||
|
||||
except Exception as e:
|
||||
task_name = self.img_list[0].task_name
|
||||
print(f"{task_name}, 拼接图片失败：{e}")
|
||||
return None
|
||||
|
||||
def save_long_image(self):
|
||||
long_image = self.concatenate_images_vertically() # 垂直拼接长图片
|
||||
long_image.save(f"output/{self.img_list[0].task_name}.png") # 保存长图到本地
|
||||
@ -1,5 +1,4 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from web_img import WebImg
|
||||
|
||||
|
||||
class Parser:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user