PHP Crawler - Sitemap

public function __construct($url) {
     global $db;  

     $this->mark= $this->getContent($url); 


  }


public function getContent($url) {
    return file_get_contents($url);  
  }


protected function GetLinks() {
    if (!empty($this->mark)){
      preg_match_all('/\s+href\s*=\s*[\'\']?([^\s\'\']+)[\'\'\s]+/ims', $this->markup, $links, PREG_PATTERN_ORDER);
      foreach($links as $key => $val) {
      return !empty($links[1]) ? $links[1] : FALSE;
    }
  }

<?
 // Php.ini Set
 error_reporting(E_ALL);
 ini_set('display_errors', 1); 
 
 // Include
 require("../../include/config.php"); 
 require("../../include/db.class");
 $db = new db($sqlhost,$sqldb,$sqluser,$sqlpassword);


 require("class.php");


 // Config für Crawler
 $MAX_DEPP = 6;
 $ALLOW_EXT = array('html', 'htm', 'xhtml', 'php', 'jpg');
?>

<?
 
 /* Enthält die Funktionen fürs crawlen */


  // SET URL die 30 Tage alt sind auf 0
  function UpdateUrls() {
      global $db;
      $db->actionQuery("UPDATE crawler SET crawled='0' WHERE date<".time()."-2592000");
   }

  // Holt eine URL aus der DB
  function getURLToCrawl($pageid) {
      global $db;
      $url = $db->singleQuery("SELECT id FROM crawler WHERE crawled = '0' AND page_id='".$pageid."'");
      $id = $url['id'];
      return $id;
   }

  // Prüft ob URL erreichbar
  function doUrl($URL) {
      global $ALLOW_EXT, $fullurl;
      $handle = @fopen ($URL, "r");
      if ($handle === false) return false;

      $fnpath = substr($URL, strlen($fullurl));
      if ($fnpath !== false) {
         $fnsplit = explode(".", $fnpath);
         $fnext = $fnsplit[count($fnsplit)-1];
         if (!in_array($fnext, $ALLOW_EXT) && strlen($fnext) <= 5 && strpos($fnext, '/') === false) return false;
      }

      $buffer = "";
      while (!feof ($handle)) {
         $buffer .= fgets($handle, 4096);
      }
      fclose ($handle);

      return $buffer;
   }

  // Holt alle URLS von der Seite
  function getUrls($crowl_page, $deep = 0) {
      global $MAX_DEPP;
      if ($deep >= $MAX_DEPP) return array();
      $match_url = array();
      $re_expresion = "/\s+href\s*=\s*[\'\']?([^\s\'\']+)[\'\'\s]+/ims";
      preg_match_all ($re_expresion, $crowl_page, $match_url, PREG_PATTERN_ORDER);
      return $match_url[1];
   }

  // Parsed eine URL richtig
  function parseURL($URL_to_parse, $fullurl, $aktive_URL) {
      if (strpos ($URL_to_parse, "://") != 0 && substr($URL_to_parse, 0, 7) != "http://") return false;
      if (substr($URL_to_parse, 0, 1) != "/" && substr($URL_to_parse, 0, 7) != "http://") $URL_to_parse = $aktive_URL . "/" . $URL_to_parse;
      if (substr($URL_to_parse, 0, 7) != "http://") $URL_to_parse = $fullurl . "/" . $URL_to_parse;
      if(strpos($URL_to_parse,"#")!==false){$URL_to_parse = "";}
      if(strpos($URL_to_parse,"javascript")!==false){$URL_to_parse = "";}

      $URL_to_parse = str_replace("/./", "/", $URL_to_parse);
      $URL_to_parse = preg_replace("/\/[\/]+/i", "/", $URL_to_parse);
      $URL_to_parse = str_replace("http:/", "http://", $URL_to_parse);
      $URL_to_parse = str_replace("&amp;", "&", $URL_to_parse);
      $URL_to_parse = preg_replace("/sid=[\w\d]+/i", "", $URL_to_parse);

      return $URL_to_parse;
   }

   // Durchgeht alle URLS
   function cleanURLs($get_URLs, $fullurl, $aktive_URL) {
      $URLs_cleaned = array();
      $counter = 0;
      foreach($get_URLs as $id => $URL) {
         $URL = parseURL($URL, $fullurl, $aktive_URL);
         if ($URL === false || strpos ($URL, $fullurl) !== 0) continue;
         $URLs_cleaned[$counter++] = $URL;
      }
      return $URLs_cleaned;
   }




    // Fügt eine gefunde URL zu DB hinzu
    function add($pageid, $new_urls, $deep) {
      foreach($new_urls as $id => $URL)    {
         toDB($pageid, $URL, $deep);
      }

      return 0;
   }

   // DB Funtkion
   function toDB($pageid, $URL, $deep) {
     global $db;
      $registrierte_site = $db->singleQuery("SELECT COUNT(id) FROM crawler WHERE site='".$URL."' AND page_id='".$pageid."'");
       $registrierte_site = $registrierte_site['COUNT(id)'];
      if ($registrierte_site == 0 && $URL != "") {
         $pr = "";
         $db->actionQuery("INSERT INTO crawler (site, deep, date,pr, page_id) VALUES('".$URL."','".$deep."','".time()."','".$pr."','".$pageid."')");
         return 1;
      } 
      return 0;
   }
 
?>

<?php 


set_time_limit(0);
require("global.php"); 
 
 // Seite und ID
 $fullurl = "http://www.forum-hilfe.de";
 $pageid = 2;
 
 // Prüfen ob diese Seite bereits in der DB
 $registrierte_site = $db->singleQuery("SELECT COUNT(id) FROM crawler WHERE site='".$fullurl."'  AND page_id='".$pageid."'");
 $registrierte_site = $registrierte_site['COUNT(id)'];
 
 // Soll mal eine Pagerank abfrage rein
 $pr = "";
 
 // Wenn nicht vorhanden einfügen
 if($registrierte_site == 0) {
 $db->actionQuery("INSERT INTO crawler 
            (site, deep, date,pr, page_id) VALUES('".$fullurl."','0','".time()."','".$pr."','".$pageid."')");
 }
 
 // Alle URLS die 30 Tage alt sind auf "none gecrawlt" setzen
 UpdateUrls();
 
 // ALLE URLS der Domain durchgehen, den Inhalt dursuchen und ggf. neue Links seztzen
 while($URL_ID = getURLToCrawl($pageid)) {

     // Setzt aktuelle URL auf gecrawlt
     $db->actionQuery("UPDATE crawler SET crawled='1' WHERE id='".$URL_ID."'");

     // Holt die Infos von der URL aus der DB
     $SQL = $db->singleQuery("SELECT site, deep FROM crawler WHERE id='".$URL_ID."'");
     $NEW_URL = $SQL['site']; $NEW_DEEP = $SQL['deep'];

     // Parse 
     $aktive_URL = preg_replace("/([^\/])\/[^\/]+$/i", "\\1", $NEW_URL);

     // Prüefe ob URL vorhanden
     $crawl = doUrl($NEW_URL);

     // Siete nicht vorhanden, dann löschen
     if ($crawl === false) {
       $db->actionQuery("DELETE FROM crawler WHERE id='".$URL_ID."'");
       continue;
     }

     // Inhalt laden und URL durchsuchen 
     $get_URLs = getUrls($crawl, $NEW_DEEP + 1);

     // Urls parsen
     $clean_URLs = cleanURLs($get_URLs, $fullurl, $aktive_URL);

     // Neue URLS in die DB
     $URLs_to_crawl = add($pageid, $clean_URLs, $NEW_DEEP + 1);
 }




?>