ようこそゲストさん

無能日記

メッセージ欄

分類 【R&D (feed)】 で検索

一覧で表示する

2007/03/18(日) plaggerでfeedを集め

はてブ 2007/04/09 13:33 R&D (feed)poti
plaggerでfeedを集めてみた。
なぜか、publish::CGIなるものを作ってpostするようにしてみた。
ユーザー認証して、身内同士のfeed収集コミュニティーなんか作ったら面白いかも。

ソースはこんな感じです。例によって適当!

# Plagger::Plugin::Publish::CGI
# Auther by poti

package Plagger::Plugin::Publish::CGI;
use strict;
use base qw( Plagger::Plugin );

use Digest::SHA1;
use Encode;
use LWP::UserAgent;
use HTTP::Request::Common qw(POST);
use DateTime::Duration;
use Plagger::Util;

sub register {
    my($self, $context) = @_;
    $context->register_hook(
        $self,
        'publish.feed' => \&feed,
        'publish.finalize' => \&finalize,
    )
}my $feed_count = 0;

sub feed {
    my($self, $context, $args) = @_;
    
    push @{$self->{_feeds}}, $args->{feed};
}

sub post_to_cgi {
    my ($self, $url, $timeout, $args)  = @_;
    my %args = %{$args};
    my $request = POST($url, [%args]);

    my $ua = LWP::UserAgent->new;
    $ua->timeout($timeout);
    $ua->agent('Plagger-Publish-CGI');
    my $res = $ua->request($request);
    return $res->as_string;
}
sub finalize {
    my($self, $context, $args) = @_;
    my $sha1;
    my %args = ();
    
    my $url = $self->conf->{url} || '';
    my $action = $self->conf->{action} || 'post';
    my $encode = $self->conf->{encode} || 'utf8';
    my $timeout = $self->conf->{timeout} || '180';
    my $username = $self->conf->{username} || '';
    my $password = $self->conf->{password} || '';
    my $postid = $self->conf->{postid} || '';
    my $comment = $self->conf->{comment} || '';

    $args{"UserName"}= $username;

    $sha1 = Digest::SHA1->new;
    $sha1->add($username."@".$password);
    $args{"Password"} = $sha1->hexdigest;

    $args{"Action"} = $action;
    $args{"PostID"} = $postid;
    $args{"Comment"} = $postid;
   foreach my $feed ( @{$self->{_feeds}} ) {
        $sha1 = Digest::SHA1->new;
        $sha1->add($feed->title);
        $args{"FeedHash"} = $sha1->hexdigest;
        $args{"FeedTitle"} = encode($encode, $feed->title);

        foreach my $entry (@{ $feed->entries }) {
            $sha1 = Digest::SHA1->new;
            $sha1->add($entry->title);
            $args{"EntryHash"} = $sha1->hexdigest;
            $args{"EntryTitle"} = encode($encode, $entry->title);
            $args{"BodyText"} = encode($encode, $entry->body);
            my $date = $entry->date || Plagger::Date->now(timezone => $context->conf->{timezone});
            $args{"Date"} = $date->strftime('%Y/%m/%d %H:%M:%S');
            $args{"Link"} = $entry->link || '';
        }
        eval {
            my $res = $self->post_to_cgi($url, $timeout, \%args);
            $context->log(info => "Successfuly posted: $res");
        }; if (my $err = $@) {
            $err = $err->[0] if ref $err && ref $err eq 'ARRAY';
            $context->error($err);
        }
    }
}

1;

__END__

=head1 NAME

Plagger::Plugin::Publish::CGI - Publish as post to CGI

=head1 SYNOPSIS

  - module: Publish::CGI
    config:
      url: http://www.poti.net/plagger.cgi
      timeout: 180
      username: poti
      password: himitu
      postid: poti
      comment: "by poti"

=head1 DESCRIPTION

This plugin publishes feeds as post to CGI.

=head1 CONFIG

=over 6

=item url

cgi url for post

=item timeout

timeout

=item username

username

=item password

password

=item postid

identifier as nickname

=item comment

comment by free text

=back

=head1 AUTHOR

poti

=head1 SEE ALSO

L<Plagger>, L<Encode::Supported>, L<Digest::SHA1>, L<LWP::UserAgent>, L<HTTP::Request::Common>

=cut


でもって受け側は以下。そして作りかけ!

#!/usr/bin/python
import cgi
import string
import sys
import os
import re
import time
from stat import *

URL="http://enjoy.potix.jp/igather/gather.cgi"
BaseDir="entries"

ContentTypeOutputFlag = False
KeyList = ["UserName", "Password", "PostID", "Comment", 
           "FeedHash", "FeedTitle", "EntryHash", "EntryTitle",
           "Category", "BodyText", "Action", "Target", 
           "Date", "Link", "PrevDate", "Option"]
DataInfo = {}
colors = [ "#ffd0d0", "#d0ffd0", "#d0d0ff", "#ffffd0", "#ffd0ff", "#d0ffff" ]
colorIndex = 0;

##-- output content-type header -- ##
def PrintConterntType(TypeFlag):
    global ContentTypeOutputFlag
    if ContentTypeOutputFlag == False:
        if TypeFlag.lower() == "html":
            print "Content-Type: text/html; charset=utf8\r\n",
        elif TypeFlag.lower() == "plain":
            print "Content-Type: text/plain\r\n",
        elif TypeFlag.lower() == "png":
            print "Content-Type: image/png\r\n",
        elif TypeFlag.lower() == "gif":
            print "Content-Type: image/gif\r\n",
        ContentTypeOutputFlag = True
        return True
    else:
        return False

##-- get data from key list --##
def GetReqFromKeyList(cgiReq, KeyList=[]):
    GetReq = {}
    for key in KeyList:
        GetReq[key] = cgiReq.getvalue(key, "").strip()
    return GetReq

##-- print html head--""
def PrintHTMLHead():
    print "<html><head><title>feed list</title>"
    print "<meta http-equiv='Content-Type' content='text/html;charset=utf-8'>"
    print "<style type='text/css'>"
    print "table.borderr { border: solid 1px #000000; border-collapse: separate; }"
    print "td.border { border: solid 1px #c0c0c0; empty-cells: show; }"
    print "</style>"
    SCRIPT = '''
    <script type="text/javascript">
    <!--

    var xmlhttp;

    function createXMLHttpRequest() {
        if (window.XMLHttpRequest) {
            return new XMLHttpRequest();
        } else if (window.ActiveXObject) {
            try {
                return new ActiveXObject("Msxml2.XMLHTTP");
            } catch (e) {
                try {
                    return new ActiveXObject("Microsoft.XMLHTTP");
                } catch (e2) {
                    return null;
                }
            }
        } else {
            return null;
        }
    }
    function postRequest(url, requestData, id) {
        var xmlhttp = createXMLHttpRequest();

        function resCheck() {
            if (xmlhttp.readyState == 4 && xmlhttp.status == 200) {
                var element = document.getElementById("status_" + id);
                element.innerHTML = "read";
            }
        }
        if (xmlhttp) {
            xmlhttp.onreadystatechange = resCheck;
            xmlhttp.open('POST', url, true);
            xmlhttp.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
            xmlhttp.send(requestData);
        }
    }

    function Marking(url, target, option) {
        var params = new Array();
        params.push("Target=" + target);
        params.push("Option=" + option);
        postRequest(url, params.join("&"), target);
    }

    // -->
   </script>
    '''
    print SCRIPT
    print "</head><body>"
##-- print html fut--""
def PrintHTMLFut():
    print "</body><html>"

##-- user validate --##
def Validate(User, Passwd):
    return True

##-- GetText --##
def GetText(file):
    fileobj = open(file, "rb")
    Text = fileobj.read()
    fileobj.close()
    return Text
##-- Print Index--##
def PrintIndex(URL, feed, entry, root, date, link, read):
    parse_fail = False
    global colors;
    global colorIndex;

    if read == "Yes":
        status = "read"
    else:
        status = ""
    items = root.split("/")
    if len(items) != 3:
       parse_fail = True
    if (parse_fail):
        if feed:
            print "<tr><th class='border' colspan='4' bgcolor='%s' >%s</th></tr>" % (colors[colorIndex], feed)
            colorIndex += 1
            if colorIndex == len(colors):
                colorIndex = 0;
            print "<tr><td class='border'>status</td><td class='border'>date</td><td class='border'>entries</td>
<td class='border'>backup</td></tr>"
        print "<tr><td class='border'>%s</td> <td class='border'>%s</td> <td class='border'><a href='%s' target=
'_blank'>%s</a></td> <td class='border'>backup</td></tr>" % (status, date, link, entry)
    else:
        (basedir, feedhash, entryhash) = tuple(items)
    if feed:
        print "<tr><th class='border' colspan='4' bgcolor='%s' >%s</th></tr>" % (colors[colorIndex], feed)
        colorIndex += 1
        if colorIndex == len(colors):
            colorIndex = 0;
        print "<tr><td class='border'>status</td><td class='border'>date</td><td class='border'>entries</td><td 
class='border'>backup</td></tr>"
    print "<tr><td class='border' id='status_%s/%s'>%s</td> <td class='border'>%s</td> <td class='border'><a hre
f='%s' target='_blank' OnClick='Marking(\"%s\", \"%s/%s\", \"marking\");'>%s</a></td> <td class='border'><a href
='%s?Target=%s/%s' target='_blank'>backup</a></td></tr>" % (feedhash, entryhash, status, date, link, URL, feedha
sh, entryhash, entry, URL, feedhash, entryhash)
    
##-- view body --##
def ViewBody(Dir, URL, FeedHash, EntryHash, CheckOnly = False):
    mode = os.F_OK | os.R_OK
    path = "%s/%s" % (Dir, FeedHash)
    if os.access("%s/FeedTitle" % path, mode) == False:
       return False
    else:
       feed_title = GetText("%s/FeedTitle" % path)
    path = "%s/%s/%s" % (Dir, FeedHash, EntryHash)

    if not CheckOnly:
        if os.access("%s/EntryTitle" % path, mode) == False:
           return False
        else:
           entry_title = GetText("%s/EntryTitle" % path)
        if os.access("%s/EntryBody" % path, mode) == False:
           return False
        else:
           entry_body = GetText("%s/EntryBody" % path)
           entry_body = re.sub("\r\n|\n", "<br>", entry_body) 

        Response()
        PrintHTMLHead()
        print "<h2>%s</h2><h3>%s</h3>%s" % (feed_title, entry_title, entry_body)
        PrintHTMLFut()
    else:
        Response(False, "read")

    fileobj = open("%s/Read" % path , "w+b")
    fileobj.write("Yes")
    fileobj.close()
##-- view Index --##
def ViewIndex(Dir, URL, PrevDate):
    feed_clear = False
    entry_clear = False
    now = long(time.time())
    list = os.walk(Dir)
    Response()
    PrintHTMLHead()
    print "<table border='1' class='border'>"
    for root, dirs, files in list:
       if dirs != [] and files != []:
           feed_title = GetText("%s/FeedTitle" % root)
           feed_clear = True
       elif dirs == [] and files != []:
           date = GetText("%s/Date" % root)
           mtime = time.mktime(time.strptime(date, "%Y/%m/%d %H:%M:%S"))
           if (now - (PrevDate * 60 * 60 * 24)) < mtime:
               entry_title = GetText("%s/EntryTitle" % root)
               link = GetText("%s/Link" % root)
               read = GetText("%s/Read" % root)
               entry_clear = True
       if feed_clear and entry_clear:
           PrintIndex(URL, feed_title, entry_title, root, date, link, read)
           feed_clear = False
           entry_clear = False
       elif entry_clear:
           PrintIndex(URL, "", entry_title, root, date, link, read)
           entry_clear = False
    print "</table>"
    PrintHTMLFut()

##-- view --##
def View(Dir, URL, FeedHash = "", EntryHash = "", PrevDate = 2, CheckOnly = False):
    if FeedHash and EntryHash:
         if ViewBody(Dir, URL, FeedHash, EntryHash, CheckOnly) == False:
            Response(True, "not found target %s/%s" % (FeedHash, EntryHash))
    else:
         ViewIndex(Dir, URL, PrevDate)

##-- store post data--##
def Store(Dir, Info):
    mode = os.F_OK
    if os.access(Dir, mode) == False:
       Response(True, "can't access base directory %s" % Dir)
    path = "%s/%s" % (Dir, Info["FeedHash"])
    if os.access(path,  mode) == False:
       os.mkdir(path , 0777)
       fileobj = open("%s/FeedTitle" % path , "w+b")
       fileobj.write(DataInfo["FeedTitle"])
       fileobj.close()
    path = "%s/%s/%s" % (Dir, Info["FeedHash"],  Info["EntryHash"])
    if os.access(path , mode) == False:
       os.mkdir(path , 0777)
       fileobj = open("%s/EntryTitle" % path, "w+b")
       fileobj.write(DataInfo["EntryTitle"])
       fileobj.close()
       fileobj = open("%s/EntryBody" % path, "w+b")
       fileobj.write(DataInfo["BodyText"])
       fileobj.close()
       fileobj = open("%s/Date" % path , "w+b")
       fileobj.write(DataInfo["Date"])
       fileobj.close()
       fileobj = open("%s/Link" % path , "w+b")
       fileobj.write(DataInfo["Link"])
       fileobj.close()
       fileobj = open("%s/Read" % path , "w+b")
       fileobj.write("No")
       fileobj.close()
##-- return response --##
def Response(error = False, message = ""):
    print "Connection: close\r\n",
    PrintConterntType("html")
    if error:
       print "Status: 500 Error\r\n",
    else:  
       print "Status: 200 OK\r\n",
    print "Pragma: no-cache\r\n\r\n",
    print message

#----- get cgi request -----------#
ReqData = cgi.FieldStorage()
DataInfo = GetReqFromKeyList(ReqData, KeyList)

if DataInfo["PrevDate"] == "":
    DataInfo["PrevDate"] = 2

if DataInfo["Action"] == "":
    if DataInfo["Target"] == "":
        View(BaseDir, URL, "", "", DataInfo["PrevDate"])
    else:
        CheckOnly = False
        if DataInfo["Option"] == "marking":
            CheckOnly = True
        items = DataInfo["Target"].split("/")
        if len(items) != 2:
            Response(True, "invalid target parameter")
            sys.exit(1)
        (feedhash, entryhash) = tuple(items)
        View(BaseDir, URL, feedhash, entryhash, DataInfo["PrevDate"], CheckOnly)
elif DataInfo["Action"] == "post":
    if Validate(DataInfo["UserName"], DataInfo["Password"]) == False:
        Response(True, "invalid user")
        sys.exit(1)
    Store(BaseDir, DataInfo)
    Response(False, "accept post data")