torah.py

import logging
logger = logging.getLogger(__name__)

import os
import re
from deep_translator import GoogleTranslator
from deep_translator import exceptions
from gematria import calculate_gematria
import math
import xml.etree.ElementTree as ET
import glob

# Hebrew letters and whitespace range
DIACRITICS_REGEX = re.compile(r"[^\u05D0-\u05EA\s]+")

def process_json_files(start, end, step, rounds="1", length=0,
                       tlang="en",
                       strip_spaces=True,
                       strip_in_braces=True,
                       strip_diacritics=True,
                       translate=False):
    base_path = "texts/tanach"
    translator = GoogleTranslator(source='auto', target=tlang)
    results = []

    for i in range(start, end + 1):
        file_pattern = f"{base_path}/{i:02}*.xml"
        matching_files = glob.glob(file_pattern)

        if not matching_files:
            results.append({"error": f"No file matching pattern '{file_pattern}' found."})
            continue

        for file_name in matching_files:
            try:
                tree = ET.parse(file_name)
                root = tree.getroot()

                # Step 1: Collect text by verse, each verse's words joined by spaces
                text_blocks = []
                for verse in root.findall('.//v'):
                    verse_words = []
                    for word in verse.findall('./w'):
                        verse_words.append("".join(word.itertext()))
                    # Now join words into a single verse string
                    verse_text = " ".join(verse_words)
                    text_blocks.append(verse_text)

                # Step 2: Join all verses with spaces to get a single string
                full_text = " ".join(text_blocks)

                # -- The rest of the cleaning logic remains as before --
                if strip_in_braces:
                    full_text = re.sub(r"\[.*?\]", "", full_text, flags=re.DOTALL)
                if strip_diacritics:
                    full_text = DIACRITICS_REGEX.sub("", full_text)
                if strip_spaces:
                    # Remove *all* spaces
                    full_text = full_text.replace(" ", "")
                else:
                    # Collapse multiple spaces into one
                    full_text = re.sub(r'\s+', ' ', full_text)

                text_length = len(full_text)
                if text_length == 0:
                    # If after cleaning, there's no text, skip
                    continue

                rounds_list = list(map(float, rounds.split(',')))  # Allow floats
                result_text = ""

                # -- Rounds logic (unchanged) --
                for r in rounds_list:
                    abs_r = abs(r)
                    full_passes = math.floor(abs_r)
                    remainder = abs_r - full_passes

                    base_chars = text_length // step

                    if base_chars == 0:
                        if abs_r > 1:
                            chars_per_full_pass = 1
                        else:
                            chars_per_full_pass = 0
                        chars_for_remainder = 0
                    else:
                        chars_per_full_pass = base_chars
                        chars_for_remainder = math.floor(base_chars * remainder)

                    if r > 0:
                        current_index = (step - 1) % text_length
                        direction = 1
                    else:
                        current_index = (text_length - step) % text_length
                        direction = -1

                    pass_result = ""
                    for pass_num in range(1, full_passes + 1):
                        current_pass_chars = ""
                        for _ in range(chars_per_full_pass):
                            if chars_per_full_pass == 0:
                                break
                            current_pass_chars += full_text[current_index]
                            current_index = (current_index + direction * step) % text_length
                        # Keep only the last full pass
                        if pass_num == full_passes:
                            pass_result = current_pass_chars

                    if remainder > 0 and chars_for_remainder > 0:
                        current_pass_chars = ""
                        for _ in range(chars_for_remainder):
                            current_pass_chars += full_text[current_index]
                            current_index = (current_index + direction * step) % text_length
                        pass_result += current_pass_chars

                    result_text += pass_result

                # Optional translation
                translated_text = ""
                if translate and result_text:
                    translated_text = translator.translate(result_text)

                if length != 0:
                    result_text = result_text[:length]

                # Append final if we have any picked text
                if result_text:
                    title_el = root.find('.//names/name')
                    title_str = title_el.text if title_el is not None else os.path.basename(file_name)
                    results.append({
                        "book": i,
                        "title": title_str,
                        "result_text": result_text,
                        "result_sum": calculate_gematria(result_text),
                        "translated_text": translated_text,
                        "source_language": "he",
                    })

            except FileNotFoundError:
                results.append({"error": f"File {file_name} not found."})
            except ET.ParseError as e:
                results.append({"error": f"File {file_name} could not be read as XML: {e}"})
            except KeyError as e:
                results.append({"error": f"Expected key 'text' is missing in {file_name}: {e}"})

    logger.debug(f"Returning results from torah.process_json_files: {results}")
    return results


# Tests
test_results = [
    (process_json_files(0, 0, 21, rounds="3", length=0), "ק"),
    (process_json_files(0, 0, 22, rounds="1", length=0), "ת"),
    (process_json_files(0, 0, 22, rounds="3", length=0), "ת"),
    (process_json_files(0, 0, 23, rounds="3", length=0), "ג"),
    (process_json_files(0, 0, 11, rounds="1", length=0), "כת"),
    (process_json_files(0, 0, 2, rounds="1", length=0), "בדוחילנעצרת"),
    (process_json_files(0, 0, 23, rounds="1", length=0), None),  # Expect None, when no results
    (process_json_files(0, 0, 23, rounds="-1", length=0), None),  # Expect None, when no results
    (process_json_files(0, 0, 22, rounds="-1", length=0), "א"),
    (process_json_files(0, 0, 22, rounds="-2", length=0), "א"),
    (process_json_files(0, 0, 1, rounds="1,-1", length=0), "אבגדהוזחטיכלמנסעפצקרשתתשרקצפעסנמלכיטחזוהדגבא"), # Combined rounds
    (process_json_files(0, 0, 1, rounds="-1", length=0), "תשרקצפעסנמלכיטחזוהדגבא"), # Reversed Hebrew alphabet
    (process_json_files(0, 0, 1, rounds="-1.5", length=0), "תשרקצפעסנמלכיטחזוהדגבאתשרקצפעסנמל"), # Fractional rounds
]

all_tests_passed = True
for result, expected in test_results:
    result_text = result[0]['result_text'] if result else None
    if expected is None:  # Check if no result is expected
        if not result:
            logger.warning(f"Test passed: Expected no results, got no results.")
        else:
            logger.error(f"Test failed: Expected no results, but got: {result_text}")
            all_tests_passed = False
    else:
        # Check if result is not empty before accessing elements
        if result:
            if result_text == expected:
                logger.warning(f"Test passed: Expected '{expected}', got '{result_text}'")
            else:
                logger.error(f"Test failed: Expected '{expected}', but got '{result_text}'")
                all_tests_passed = False
        else:
            logger.error(f"Test failed: Expected '{expected}', but got no results")
            all_tests_passed = False

if all_tests_passed:
    logger.info("All round tests passed.")