Zend FR

require_once 'Zend/Search/Lucene.php';
require_once 'Zend/Http/Client.php';
require_once 'Zend/Log.php';
require_once 'Zend/Log/Writer/Stream.php';
// Define some constants
define('APP_ROOT', realpath(dirname(dirname(__FILE__))));
define('START_URI', 'http://example.com/blog');
define('MATCH_URI', 'http://example.com/blog');
// Set up log
$log = new Zend_Log(new Zend_Log_Writer_Stream(APP_ROOT . DIRECTORY_SEPARATOR .
'var' . DIRECTORY_SEPARATOR . 'log' . DIRECTORY_SEPARATOR . 'crawler.log'));
$log->info('Crawler starting up');
// Set up Zend_Http_Client
$client = new Zend_Http_Client();
$client->setConfig(array('timeout' => 30));
// Open a Lucene index, or create it if it does not exist
// First, try opening

try {
$index = Zend_Search_Lucene::open('/data/index');
// If can't open, try creating
} catch (Zend_Search_Lucene_Exception $e) {
try {
$index = Zend_Search_Lucene::create('/data/index');
} catch(Zend_Search_Lucene_Exception $e) {
// If both fail, give up and show error message
echo "Unable to open or create index: {$e->getMessage()}";
}
}
// Open index
$indexpath =  APP_ROOT . DIRECTORY_SEPARATOR . 'var' .
DIRECTORY_SEPARATOR . 'index';
 
try {
$index = Zend_Search_Lucene::open($indexpath);
$log->info("Opened existing index in". $indexpath);
// If can't open, try creating
} catch (Zend_Search_Lucene_Exception $e) {
try {
$index = Zend_Search_Lucene::create($indexpath);
$log->info("Created new index in ". $indexpath);
// If both fail, give up and show error message
} catch(Zend_Search_Lucene_Exception $e) {
$log->error("Failed opening or creating index in ". $indexpath);
$log->error($e->getMessage());
echo "Unable to open or create index: {$e->getMessage()}";
exit(1);
}


}


$contents = file_get_contents($file);
// Create a new document object
$doc = new Zend_Search_Lucene_Document();
$doc->addField(Zend_Search_Lucene_Field::UnStored('body', $contents));
$doc->addField(Zend_Search_Lucene_Field::UnIndexed('file', $file));
// Add document to index
$index->addDocument($doc);
// Index an HTML file
$doc = Zend_Search_Lucene_Document_Html::loadHTMLFile($filename);
$index->addDocument($doc);
// Index an HTML string, storing the body content in the index
$doc = Zend_Search_Lucene_Document_Html::loadHTML($htmlString, true);
$index->addDocument($doc);
// Set up the targets array
$targets = array(START_URI);
// Start iterating
for($i = 0; $i < count($targets); $i++) {
// Fetch content with HTTP Client
$client->setUri($targets[$i]);
$response = $client->request();
if ($response->isSuccessful()) {
$body = $response->getBody();
$log->info("Fetched " . strlen($body) . " bytes from {$targets[$i]}");
// Create document
$doc = Zend_Search_Lucene_Document_Html::loadHTML($body);
// Index
$index->addDocument($doc);
$log->info("Indexed {$targets[$i]}");
// ... contd in next slide ...

// ... contd from previous slide ...
// Fetch new links
$links = $doc->getLinks();
foreach ($links as $link) {
if ((strpos($link, MATCH_URI) !== false) &&
(! in_array($link, $targets))) $targets[] = $link;
}
} else {
$log->warning("Requesting $url returned HTTP " .
$response->getStatus());
}
}
$log->info("Iterated over " . count($targets) . " documents");
$log->info("Optimizing index...");
$index->optimize();
$index->commit();
$log->info("Done. Index now contains " . $index->numDocs() . " documents");
$log->info("Crawler shutting down");
// ... patching after fetching content
// Create document
$body_checksum = md5($body);
$doc = Zend_Search_Lucene_Document_Html::loadHTML($body);
$doc->addField(Zend_Search_Lucene_Field::UnIndexed('url', $targets[$i]));
$doc->addField(Zend_Search_Lucene_Field::UnIndexed('md5', $body_checksum));
// Index
$index->addDocument($doc);
$log->info("Indexed {$targets[$i]}");
// ... continue to fetch new links
// $path is the path of a file we want to reindex
$hits = $index->find('path:' . $path);
foreach ($hits as $hit) {
$index->delete($hit->id);
}
// ... patching before creating the document
// See if document exists and needs reindexing
$body_checksum = md5($body);
$hits = $index->find('url:' . $targets[$i]);
$matched = false;
foreach ($hits as $hit) {
if ($hit->md5 == $body_checksum) {
if ($matched == true) $index->delete($hit->id);
$matched = true;
} else {
$log->info($targets[$i] . " is out of date and needs reindexing");
$index->delete($hit);
}
}
if ($matched) {
$log->info($targets[$i] . " is up to date, skipping");
continue;
}

<?php
require_once 'Zend/Controller/Action.php';
class SearchController extends Zend_Controller_Action
{
public function indexAction()
{
$query = $this->getRequest()->getParam('q');
if ($query) {
require_once 'Zend/Search/Lucene.php';
$index = Zend_Search_Lucene::open(APP_ROOT . '/var/index');
$hits = $index->find($query);
$view = $this->initView();
$view->query = $query;
if (! empty($hits)) {
$view->results = $hits;
}
}
$this->render();
}
}