<?php
// includes/class-vmo-content-crawler.php
if (!defined('ABSPATH')) exit;

/**
 * VMO Content Crawler v1.0.0
 * 
 * Crawls website content to build knowledge base for AI phone assistant.
 * Extracts content from pages, posts, and products for contextual responses.
 */

class VMO_Content_Crawler {
    
    private $max_pages = 50;
    private $max_content_length = 2000;
    
    public function __construct() {
        add_action('init', array($this, 'schedule_crawl_tasks'));
    }
    
    /**
     * Schedule automatic crawling tasks
     */
    public function schedule_crawl_tasks() {
        if (!wp_next_scheduled('vmo_daily_content_crawl')) {
            wp_schedule_event(time(), 'daily', 'vmo_daily_content_crawl');
        }
        
        add_action('vmo_daily_content_crawl', array($this, 'crawl_all_users_content'));
    }
    
    /**
     * Crawl content for all users with phone service
     */
    public function crawl_all_users_content() {
        global $wpdb;
        
        $phone_table = $wpdb->prefix . 'vmo_phone_numbers';
        // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching
        $active_users = $wpdb->get_col(
            "SELECT DISTINCT user_id FROM {$phone_table} WHERE status = 'active'"
        );
        
        foreach ($active_users as $user_id) {
            $this->crawl_user_content($user_id);
        }
    }
    
    /**
     * Crawl and process content for specific user
     */
    public function crawl_user_content($user_id) {
        // ✅ TOKEN DEDUCTION v8.1.0: Charge for daily content embeddings (2,000 tokens)
        // This builds the AI knowledge base for the phone assistant
        if (function_exists('vmo_deduct_tokens')) {
            $cost = defined('VMO_TOKENS_CONTENT_EMBEDDINGS') ? VMO_TOKENS_CONTENT_EMBEDDINGS : 2000;
            if (!vmo_deduct_tokens($user_id, $cost, 'content_embeddings', 'Daily AI knowledge base update')) {
                error_log("VMO Content Crawler: Insufficient tokens for user {$user_id}");
                return new WP_Error('insufficient_tokens', 'Insufficient tokens for content embedding');
            }
        }
        
        $website_url = $this->get_user_website($user_id);
        
        if (empty($website_url)) {
            return new WP_Error('no_website', 'No website URL found for user');
        }
        
        // Get content from various sources
        $content_data = [];
        
        // Crawl main pages
        $content_data = array_merge($content_data, $this->crawl_main_pages($website_url));
        
        // Crawl blog posts
        $content_data = array_merge($content_data, $this->crawl_blog_posts($website_url));
        
        // Crawl product pages (if e-commerce)
        $content_data = array_merge($content_data, $this->crawl_products($website_url));
        
        // Crawl services/portfolio
        $content_data = array_merge($content_data, $this->crawl_services($website_url));
        
        // Process and clean content
        $processed_content = $this->process_content($content_data);
        
        // Create embeddings using Voice Processor
        $voice_processor = new VMO_Voice_Processor();
        $embeddings = $voice_processor->create_embeddings($processed_content);
        
        if (!is_wp_error($embeddings)) {
            $voice_processor->save_knowledge_base($user_id, $embeddings);
            
            // Update crawl timestamp
            update_user_meta($user_id, 'vmo_last_content_crawl', current_time('mysql'));
            
            return [
                'success' => true,
                'pages_crawled' => count($content_data),
                'embeddings_created' => count($embeddings)
            ];
        }
        
        return $embeddings; // Return error if embeddings failed
    }
    
    /**
     * Get user's website URL
     */
    private function get_user_website($user_id) {
        // Try user meta first
        $website = get_user_meta($user_id, 'vmo_website_url', true);
        
        if (empty($website)) {
            // Fallback to current site URL
            $website = home_url();
        }
        
        return $website;
    }
    
    /**
     * Crawl main pages (About, Contact, Services, etc.)
     */
    private function crawl_main_pages($base_url) {
        $main_pages = [
            'about', 'about-us', 'services', 'contact', 'contact-us',
            'portfolio', 'work', 'team', 'pricing', 'faq', 'location'
        ];
        
        $content_data = [];
        
        foreach ($main_pages as $page_slug) {
            $page_url = trailingslashit($base_url) . $page_slug;
            $content = $this->fetch_page_content($page_url);
            
            if (!empty($content)) {
                $content_data[] = [
                    'text' => $content,
                    'source' => $page_slug . ' page',
                    'url' => $page_url,
                    'type' => 'page'
                ];
            }
            
            // Respect server resources
            usleep(500000); // 0.5 second delay
        }
        
        return $content_data;
    }
    
    /**
     * Crawl blog posts
     */
    private function crawl_blog_posts($base_url) {
        $content_data = [];
        
        // Try common blog URLs
        $blog_urls = [
            trailingslashit($base_url) . 'blog',
            trailingslashit($base_url) . 'news',
            $base_url // Homepage might have blog
        ];
        
        foreach ($blog_urls as $blog_url) {
            $posts = $this->extract_blog_posts($blog_url);
            
            foreach ($posts as $post) {
                if (count($content_data) >= $this->max_pages) break;
                
                $post_content = $this->fetch_page_content($post['url']);
                
                if (!empty($post_content)) {
                    $content_data[] = [
                        'text' => $post_content,
                        'source' => 'blog post: ' . $post['title'],
                        'url' => $post['url'],
                        'type' => 'blog'
                    ];
                }
                
                usleep(500000); // 0.5 second delay
            }
        }
        
        return $content_data;
    }
    
    /**
     * Crawl product pages
     */
    private function crawl_products($base_url) {
        $content_data = [];
        
        // Try common product URLs
        $product_urls = [
            trailingslashit($base_url) . 'products',
            trailingslashit($base_url) . 'shop',
            trailingslashit($base_url) . 'store'
        ];
        
        foreach ($product_urls as $product_url) {
            $products = $this->extract_products($product_url);
            
            foreach ($products as $product) {
                if (count($content_data) >= $this->max_pages) break;
                
                $product_content = $this->fetch_page_content($product['url']);
                
                if (!empty($product_content)) {
                    $content_data[] = [
                        'text' => $product_content,
                        'source' => 'product: ' . $product['title'],
                        'url' => $product['url'],
                        'type' => 'product'
                    ];
                }
                
                usleep(500000); // 0.5 second delay
            }
        }
        
        return $content_data;
    }
    
    /**
     * Crawl services pages
     */
    private function crawl_services($base_url) {
        $content_data = [];
        
        $services_url = trailingslashit($base_url) . 'services';
        $services = $this->extract_services($services_url);
        
        foreach ($services as $service) {
            if (count($content_data) >= $this->max_pages) break;
            
            $service_content = $this->fetch_page_content($service['url']);
            
            if (!empty($service_content)) {
                $content_data[] = [
                    'text' => $service_content,
                    'source' => 'service: ' . $service['title'],
                    'url' => $service['url'],
                    'type' => 'service'
                ];
            }
            
            usleep(500000); // 0.5 second delay
        }
        
        return $content_data;
    }
    
    /**
     * Fetch and clean content from a page
     */
    private function fetch_page_content($url) {
        $response = wp_remote_get($url, [
            'timeout' => 15,
            'user-agent' => 'VMO Content Crawler/1.0'
        ]);
        
        if (is_wp_error($response)) {
            return '';
        }
        
        $status_code = wp_remote_retrieve_response_code($response);
        if ($status_code !== 200) {
            return '';
        }
        
        $html = wp_remote_retrieve_body($response);
        
        if (empty($html)) {
            return '';
        }
        
        return $this->extract_text_content($html);
    }
    
    /**
     * Extract clean text content from HTML
     */
    private function extract_text_content($html) {
        // Remove script and style elements
        $html = preg_replace('/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/mi', '', $html);
        $html = preg_replace('/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/mi', '', $html);
        
        // Extract main content areas
        $content_patterns = [
            '/<main[^>]*>(.*?)<\/main>/s',
            '/<article[^>]*>(.*?)<\/article>/s',
            '/<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)<\/div>/s',
            '/<div[^>]*id="[^"]*content[^"]*"[^>]*>(.*?)<\/div>/s'
        ];
        
        $extracted_content = '';
        
        foreach ($content_patterns as $pattern) {
            if (preg_match($pattern, $html, $matches)) {
                $extracted_content = $matches[1];
                break;
            }
        }
        
        // If no main content found, use body
        if (empty($extracted_content)) {
            if (preg_match('/<body[^>]*>(.*?)<\/body>/s', $html, $matches)) {
                $extracted_content = $matches[1];
            } else {
                $extracted_content = $html;
            }
        }
        
        // Remove navigation, header, footer, sidebar
        $removal_patterns = [
            '/<nav[^>]*>.*?<\/nav>/s',
            '/<header[^>]*>.*?<\/header>/s',
            '/<footer[^>]*>.*?<\/footer>/s',
            '/<aside[^>]*>.*?<\/aside>/s',
            '/<div[^>]*class="[^"]*sidebar[^"]*"[^>]*>.*?<\/div>/s'
        ];
        
        foreach ($removal_patterns as $pattern) {
            $extracted_content = preg_replace($pattern, '', $extracted_content);
        }
        
        // Convert to plain text
        $text = strip_tags($extracted_content);
        
        // Clean up whitespace
        $text = preg_replace('/\s+/', ' ', $text);
        $text = trim($text);
        
        // Limit content length
        if (strlen($text) > $this->max_content_length) {
            $text = substr($text, 0, $this->max_content_length);
            $text = substr($text, 0, strrpos($text, ' ')) . '...';
        }
        
        return $text;
    }
    
    /**
     * Extract blog posts from blog page
     */
    private function extract_blog_posts($blog_url) {
        $html = $this->fetch_raw_html($blog_url);
        
        if (empty($html)) {
            return [];
        }
        
        $posts = [];
        
        // Common blog post link patterns
        $patterns = [
            '/<article[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*>([^<]+)<\/a>.*?<\/article>/s',
            '/<h[2-6][^>]*><a[^>]*href="([^"]+)"[^>]*>([^<]+)<\/a><\/h[2-6]>/i',
            '/<a[^>]*href="([^"]+)"[^>]*class="[^"]*post[^"]*"[^>]*>([^<]+)<\/a>/i'
        ];
        
        foreach ($patterns as $pattern) {
            if (preg_match_all($pattern, $html, $matches, PREG_SET_ORDER)) {
                foreach ($matches as $match) {
                    $url = $this->resolve_url($match[1], $blog_url);
                    $title = strip_tags($match[2]);
                    
                    if (!empty($url) && !empty($title)) {
                        $posts[] = ['url' => $url, 'title' => $title];
                    }
                }
                break; // Use first pattern that matches
            }
        }
        
        return array_slice($posts, 0, 10); // Limit to 10 posts
    }
    
    /**
     * Extract products from shop page
     */
    private function extract_products($shop_url) {
        $html = $this->fetch_raw_html($shop_url);
        
        if (empty($html)) {
            return [];
        }
        
        $products = [];
        
        // Common product link patterns
        $patterns = [
            '/<div[^>]*class="[^"]*product[^"]*"[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*>([^<]+)<\/a>.*?<\/div>/s',
            '/<a[^>]*href="([^"]+)"[^>]*class="[^"]*product[^"]*"[^>]*>([^<]+)<\/a>/i'
        ];
        
        foreach ($patterns as $pattern) {
            if (preg_match_all($pattern, $html, $matches, PREG_SET_ORDER)) {
                foreach ($matches as $match) {
                    $url = $this->resolve_url($match[1], $shop_url);
                    $title = strip_tags($match[2]);
                    
                    if (!empty($url) && !empty($title)) {
                        $products[] = ['url' => $url, 'title' => $title];
                    }
                }
                break;
            }
        }
        
        return array_slice($products, 0, 15); // Limit to 15 products
    }
    
    /**
     * Extract services from services page
     */
    private function extract_services($services_url) {
        $html = $this->fetch_raw_html($services_url);
        
        if (empty($html)) {
            return [];
        }
        
        $services = [];
        
        // Common service link patterns
        $patterns = [
            '/<div[^>]*class="[^"]*service[^"]*"[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*>([^<]+)<\/a>.*?<\/div>/s',
            '/<a[^>]*href="([^"]+)"[^>]*class="[^"]*service[^"]*"[^>]*>([^<]+)<\/a>/i'
        ];
        
        foreach ($patterns as $pattern) {
            if (preg_match_all($pattern, $html, $matches, PREG_SET_ORDER)) {
                foreach ($matches as $match) {
                    $url = $this->resolve_url($match[1], $services_url);
                    $title = strip_tags($match[2]);
                    
                    if (!empty($url) && !empty($title)) {
                        $services[] = ['url' => $url, 'title' => $title];
                    }
                }
                break;
            }
        }
        
        return array_slice($services, 0, 10); // Limit to 10 services
    }
    
    /**
     * Fetch raw HTML from URL
     */
    private function fetch_raw_html($url) {
        $response = wp_remote_get($url, [
            'timeout' => 15,
            'user-agent' => 'VMO Content Crawler/1.0'
        ]);
        
        if (is_wp_error($response)) {
            return '';
        }
        
        $status_code = wp_remote_retrieve_response_code($response);
        if ($status_code !== 200) {
            return '';
        }
        
        return wp_remote_retrieve_body($response);
    }
    
    /**
     * Resolve relative URLs to absolute
     */
    private function resolve_url($url, $base_url) {
        if (filter_var($url, FILTER_VALIDATE_URL)) {
            return $url; // Already absolute
        }
        
        $base_parts = parse_url($base_url);
        
        if ($url[0] === '/') {
            // Absolute path
            return $base_parts['scheme'] . '://' . $base_parts['host'] . $url;
        } else {
            // Relative path
            return rtrim($base_url, '/') . '/' . ltrim($url, '/');
        }
    }
    
    /**
     * Process and clean content data
     */
    private function process_content($content_data) {
        $processed = [];
        
        foreach ($content_data as $content) {
            $text = $content['text'];
            
            // Skip if content is too short
            if (strlen($text) < 100) {
                continue;
            }
            
            // Remove excessive whitespace
            $text = preg_replace('/\s+/', ' ', $text);
            $text = trim($text);
            
            // Break long content into chunks
            if (strlen($text) > $this->max_content_length) {
                $chunks = $this->chunk_content($text, $this->max_content_length);
                
                foreach ($chunks as $i => $chunk) {
                    $processed[] = [
                        'text' => $chunk,
                        'source' => $content['source'] . ($i > 0 ? ' (part ' . ($i + 1) . ')' : ''),
                        'url' => $content['url'],
                        'type' => $content['type']
                    ];
                }
            } else {
                $processed[] = $content;
            }
        }
        
        return $processed;
    }
    
    /**
     * Chunk content into smaller pieces
     */
    private function chunk_content($text, $max_length) {
        $chunks = [];
        $sentences = preg_split('/(?<=[.!?])\s+/', $text);
        
        $current_chunk = '';
        
        foreach ($sentences as $sentence) {
            if (strlen($current_chunk . ' ' . $sentence) > $max_length) {
                if (!empty($current_chunk)) {
                    $chunks[] = trim($current_chunk);
                    $current_chunk = $sentence;
                } else {
                    // Single sentence is too long, split by words
                    $words = explode(' ', $sentence);
                    $word_chunk = '';
                    
                    foreach ($words as $word) {
                        if (strlen($word_chunk . ' ' . $word) > $max_length) {
                            if (!empty($word_chunk)) {
                                $chunks[] = trim($word_chunk);
                                $word_chunk = $word;
                            }
                        } else {
                            $word_chunk .= ($word_chunk ? ' ' : '') . $word;
                        }
                    }
                    
                    if (!empty($word_chunk)) {
                        $current_chunk = $word_chunk;
                    }
                }
            } else {
                $current_chunk .= ($current_chunk ? ' ' : '') . $sentence;
            }
        }
        
        if (!empty($current_chunk)) {
            $chunks[] = trim($current_chunk);
        }
        
        return $chunks;
    }
    
    /**
     * Manual crawl trigger for immediate updates
     */
    public function trigger_manual_crawl($user_id) {
        return $this->crawl_user_content($user_id);
    }
}
