import logging
logger = logging.getLogger(__name__)
import os
import re
from deep_translator import GoogleTranslator
from deep_translator import exceptions
from gematria import calculate_gematria
import math
import xml.etree.ElementTree as ET
import glob
# Hebrew letters and whitespace range
DIACRITICS_REGEX = re.compile(r"[^\u05D0-\u05EA\s]+")
def process_json_files(start, end, step, rounds="1", length=0,
tlang="en",
strip_spaces=True,
strip_in_braces=True,
strip_diacritics=True,
translate=False):
base_path = "texts/tanach"
translator = GoogleTranslator(source='auto', target=tlang)
results = []
for i in range(start, end + 1):
file_pattern = f"{base_path}/{i:02}*.xml"
matching_files = glob.glob(file_pattern)
if not matching_files:
results.append({"error": f"No file matching pattern '{file_pattern}' found."})
continue
for file_name in matching_files:
try:
tree = ET.parse(file_name)
root = tree.getroot()
# Step 1: Collect text by verse, each verse's words joined by spaces
text_blocks = []
for verse in root.findall('.//v'):
verse_words = []
for word in verse.findall('./w'):
verse_words.append("".join(word.itertext()))
# Now join words into a single verse string
verse_text = " ".join(verse_words)
text_blocks.append(verse_text)
# Step 2: Join all verses with spaces to get a single string
full_text = " ".join(text_blocks)
# -- The rest of the cleaning logic remains as before --
if strip_in_braces:
full_text = re.sub(r"\[.*?\]", "", full_text, flags=re.DOTALL)
if strip_diacritics:
full_text = DIACRITICS_REGEX.sub("", full_text)
if strip_spaces:
# Remove *all* spaces
full_text = full_text.replace(" ", "")
else:
# Collapse multiple spaces into one
full_text = re.sub(r'\s+', ' ', full_text)
text_length = len(full_text)
if text_length == 0:
# If after cleaning, there's no text, skip
continue
rounds_list = list(map(float, rounds.split(','))) # Allow floats
result_text = ""
# -- Rounds logic (unchanged) --
for r in rounds_list:
abs_r = abs(r)
full_passes = math.floor(abs_r)
remainder = abs_r - full_passes
base_chars = text_length // step
if base_chars == 0:
if abs_r > 1:
chars_per_full_pass = 1
else:
chars_per_full_pass = 0
chars_for_remainder = 0
else:
chars_per_full_pass = base_chars
chars_for_remainder = math.floor(base_chars * remainder)
if r > 0:
current_index = (step - 1) % text_length
direction = 1
else:
current_index = (text_length - step) % text_length
direction = -1
pass_result = ""
for pass_num in range(1, full_passes + 1):
current_pass_chars = ""
for _ in range(chars_per_full_pass):
if chars_per_full_pass == 0:
break
current_pass_chars += full_text[current_index]
current_index = (current_index + direction * step) % text_length
# Keep only the last full pass
if pass_num == full_passes:
pass_result = current_pass_chars
if remainder > 0 and chars_for_remainder > 0:
current_pass_chars = ""
for _ in range(chars_for_remainder):
current_pass_chars += full_text[current_index]
current_index = (current_index + direction * step) % text_length
pass_result += current_pass_chars
result_text += pass_result
# Optional translation
translated_text = ""
if translate and result_text:
translated_text = translator.translate(result_text)
if length != 0:
result_text = result_text[:length]
# Append final if we have any picked text
if result_text:
title_el = root.find('.//names/name')
title_str = title_el.text if title_el is not None else os.path.basename(file_name)
results.append({
"book": i,
"title": title_str,
"result_text": result_text,
"result_sum": calculate_gematria(result_text),
"translated_text": translated_text,
"source_language": "he",
})
except FileNotFoundError:
results.append({"error": f"File {file_name} not found."})
except ET.ParseError as e:
results.append({"error": f"File {file_name} could not be read as XML: {e}"})
except KeyError as e:
results.append({"error": f"Expected key 'text' is missing in {file_name}: {e}"})
logger.debug(f"Returning results from torah.process_json_files: {results}")
return results
# Tests
test_results = [
(process_json_files(0, 0, 21, rounds="3", length=0), "ק"),
(process_json_files(0, 0, 22, rounds="1", length=0), "ת"),
(process_json_files(0, 0, 22, rounds="3", length=0), "ת"),
(process_json_files(0, 0, 23, rounds="3", length=0), "ג"),
(process_json_files(0, 0, 11, rounds="1", length=0), "כת"),
(process_json_files(0, 0, 2, rounds="1", length=0), "בדוחילנעצרת"),
(process_json_files(0, 0, 23, rounds="1", length=0), None), # Expect None, when no results
(process_json_files(0, 0, 23, rounds="-1", length=0), None), # Expect None, when no results
(process_json_files(0, 0, 22, rounds="-1", length=0), "א"),
(process_json_files(0, 0, 22, rounds="-2", length=0), "א"),
(process_json_files(0, 0, 1, rounds="1,-1", length=0), "אבגדהוזחטיכלמנסעפצקרשתתשרקצפעסנמלכיטחזוהדגבא"), # Combined rounds
(process_json_files(0, 0, 1, rounds="-1", length=0), "תשרקצפעסנמלכיטחזוהדגבא"), # Reversed Hebrew alphabet
(process_json_files(0, 0, 1, rounds="-1.5", length=0), "תשרקצפעסנמלכיטחזוהדגבאתשרקצפעסנמל"), # Fractional rounds
]
all_tests_passed = True
for result, expected in test_results:
result_text = result[0]['result_text'] if result else None
if expected is None: # Check if no result is expected
if not result:
logger.warning(f"Test passed: Expected no results, got no results.")
else:
logger.error(f"Test failed: Expected no results, but got: {result_text}")
all_tests_passed = False
else:
# Check if result is not empty before accessing elements
if result:
if result_text == expected:
logger.warning(f"Test passed: Expected '{expected}', got '{result_text}'")
else:
logger.error(f"Test failed: Expected '{expected}', but got '{result_text}'")
all_tests_passed = False
else:
logger.error(f"Test failed: Expected '{expected}', but got no results")
all_tests_passed = False
if all_tests_passed:
logger.info("All round tests passed.")