63 lines
1.9 KiB
Python
63 lines
1.9 KiB
Python
# coding=utf-8
|
||
"""
|
||
@project: MaxKB
|
||
@Author：虎
|
||
@file： page_utils.py
|
||
@date：2024/11/21 10:32
|
||
@desc:
|
||
"""
|
||
from math import ceil
|
||
|
||
|
||
def page(query_set, page_size, handler, is_the_task_interrupted=lambda: False):
|
||
"""
|
||
|
||
@param query_set: 查询query_set
|
||
@param page_size: 每次查询大小
|
||
@param handler: 数据处理器
|
||
@param is_the_task_interrupted: 任务是否被中断
|
||
@return:
|
||
"""
|
||
query = query_set.order_by("id")
|
||
count = query_set.count()
|
||
for i in range(0, ceil(count / page_size)):
|
||
if is_the_task_interrupted():
|
||
return
|
||
offset = i * page_size
|
||
paragraph_list = query.all()[offset: offset + page_size]
|
||
handler(paragraph_list)
|
||
|
||
|
||
def page_desc(query_set, page_size, handler, is_the_task_interrupted=lambda: False):
|
||
"""
|
||
|
||
@param query_set: 查询query_set
|
||
@param page_size: 每次查询大小
|
||
@param handler: 数据处理器
|
||
@param is_the_task_interrupted: 任务是否被中断
|
||
@return:
|
||
"""
|
||
from common.utils.logger import maxkb_logger
|
||
|
||
query = query_set.order_by("id")
|
||
count = query_set.count()
|
||
|
||
maxkb_logger.info(f"🔍 page_desc: Processing {count} items in batches of {page_size}")
|
||
maxkb_logger.info(f"📊 Total batches to process: {ceil(count / page_size)}")
|
||
|
||
batch_count = 0
|
||
for i in sorted(range(0, ceil(count / page_size)), reverse=True):
|
||
if is_the_task_interrupted():
|
||
maxkb_logger.warning(f"⚠️ page_desc: Task interrupted during batch processing")
|
||
return
|
||
|
||
batch_count += 1
|
||
offset = i * page_size
|
||
paragraph_list = query.all()[offset: offset + page_size]
|
||
|
||
maxkb_logger.info(f"🔄 Processing batch {batch_count}/{ceil(count / page_size)}: {len(paragraph_list)} items")
|
||
handler(paragraph_list)
|
||
maxkb_logger.info(f"✅ Completed batch {batch_count}")
|
||
|
||
maxkb_logger.info(f"🎉 page_desc: Completed all {batch_count} batches")
|