author | Dan Price <dp@eng.sun.com> |
Tue, 22 Jan 2008 17:37:00 -0800 | |
changeset 239 | 20770afe33ec |
parent 220 | afceea12576c |
child 258 | b3b7592412ec |
permissions | -rw-r--r-- |
215
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
1 |
#!/usr/bin/python |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
2 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
3 |
# CDDL HEADER START |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
4 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
5 |
# The contents of this file are subject to the terms of the |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
6 |
# Common Development and Distribution License (the "License"). |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
7 |
# You may not use this file except in compliance with the License. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
8 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
9 |
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
10 |
# or http://www.opensolaris.org/os/licensing. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
11 |
# See the License for the specific language governing permissions |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
12 |
# and limitations under the License. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
13 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
14 |
# When distributing Covered Code, include this CDDL HEADER in each |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
15 |
# file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
16 |
# If applicable, add the following below this CDDL HEADER, with the |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
17 |
# fields enclosed by brackets "[]" replaced with your own identifying |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
18 |
# information: Portions Copyright [yyyy] [name of copyright owner] |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
19 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
20 |
# CDDL HEADER END |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
21 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
22 |
# Copyright 2007 Sun Microsystems, Inc. All rights reserved. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
23 |
# Use is subject to license terms. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
24 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
25 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
26 |
import os |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
27 |
import re |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
28 |
import time |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
29 |
import datetime |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
30 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
31 |
import pkg.fmri as fmri |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
32 |
import pkg.catalog as catalog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
33 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
34 |
class UpdateLogException(Exception): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
35 |
def __init__(self, args=None): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
36 |
self.args = args |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
37 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
38 |
class UpdateLog(object): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
39 |
"""The update log is a mechanism that allows clients and servers to make |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
40 |
incremental updates to their package catalogs. The server logs |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
41 |
whether it has added or removed a package, the time when the action |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
42 |
occurred, and the name of the package added or removed. The client |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
43 |
requests a list of actions that have been applied to the server's |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
44 |
catalog since a particular time in the past. The server is then able to |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
45 |
send this list of actions, allowing the client to apply these changes to |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
46 |
its catalog. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
47 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
48 |
This allows the client to obtain incremental updates to its catalog, |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
49 |
instead of having to download an entire (and largely duplicated) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
50 |
catalog each time a refresh is requested. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
51 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
52 |
The UpdateLog must have an associated catalog; however, |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
53 |
Catalogs are not required to have an UpdateLog. The UpdateLog |
220
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
54 |
allows catalogs to support incremental updates. |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
55 |
|
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
56 |
The catalog format is a + or -, an isoformat timestamp, and a catalog |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
57 |
entry in server-side format. They must be in order and separated by |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
58 |
spaces.""" |
215
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
59 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
60 |
def __init__(self, update_root, catalog, maxfiles = 336): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
61 |
"""Create an instance of the UpdateLog. "update_root" is |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
62 |
the root directory for the update log files. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
63 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
64 |
maxfiles is the maximum number of logfiles that |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
65 |
the UpdateLog will keep. A new file is added |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
66 |
for each hour in which there is an update. The |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
67 |
default value of 336 means that we keep 336 hours, or |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
68 |
14 days worth of log history.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
69 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
70 |
self.rootdir = update_root |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
71 |
self.logfd = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
72 |
self.maxfiles = maxfiles |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
73 |
self.catalog = catalog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
74 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
75 |
if not os.path.exists(update_root): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
76 |
os.makedirs(update_root) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
77 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
78 |
self._setup_logfiles() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
79 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
80 |
def __del__(self): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
81 |
"""Perform any last minute cleanup.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
82 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
83 |
if self.logfd: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
84 |
try: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
85 |
self.logfd.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
86 |
except: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
87 |
pass |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
88 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
89 |
self.logfd = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
90 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
91 |
def add_package(self, fmri, critical = False): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
92 |
"""Record that the catalog has added "fmri".""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
93 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
94 |
# First add FMRI to catalog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
95 |
ts = self.catalog.add_fmri(fmri, critical) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
96 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
97 |
# Now add update to updatelog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
98 |
self._check_logs() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
99 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
100 |
if not self.logfd: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
101 |
self._begin_log() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
102 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
103 |
if critical: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
104 |
type = "C" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
105 |
else: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
106 |
type = "V" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
107 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
108 |
logstr = "+ %s %s %s\n" % \ |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
109 |
(ts.isoformat(), type, fmri.get_fmri(anarchy = True)) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
110 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
111 |
self.logfd.write(logstr) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
112 |
self.logfd.flush() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
113 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
114 |
self.last_update = ts |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
115 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
116 |
return ts |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
117 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
118 |
def _begin_log(self): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
119 |
"""Open a log-file so that the UpdateLog can write updates |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
120 |
into it.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
121 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
122 |
filenm = time.strftime("%Y%m%d%H") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
123 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
124 |
ftime = datetime.datetime( |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
125 |
*time.strptime(filenm, "%Y%m%d%H")[0:6]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
126 |
delta = datetime.timedelta(hours=1) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
127 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
128 |
self.close_time = ftime + delta |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
129 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
130 |
self.logfd = file(os.path.join(self.rootdir, filenm), "a") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
131 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
132 |
self.logfiles.append(filenm) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
133 |
self.curfiles += 1 |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
134 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
135 |
if not self.first_update: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
136 |
self.first_update = ftime |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
137 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
138 |
def _check_logs(self): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
139 |
"""Check to see if maximum number of logfiles has been exceeded. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
140 |
If so, rotate the logs. Also, if a log is open, check to |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
141 |
see if it needs to be closed.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
142 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
143 |
if self.logfd and self.close_time < datetime.datetime.now(): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
144 |
self.logfd.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
145 |
self.logfd = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
146 |
self.close_time = 0 |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
147 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
148 |
if self.curfiles < self.maxfiles: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
149 |
return |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
150 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
151 |
excess = self.curfiles - self.maxfiles |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
152 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
153 |
to_remove = self.logfiles[0:excess] |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
154 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
155 |
for r in to_remove: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
156 |
filepath = os.path.join(self.rootdir, "%s" % r) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
157 |
os.unlink(filepath) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
158 |
self.curfiles -= 1 |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
159 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
160 |
del self.logfiles[0:excess] |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
161 |
self.first_update = datetime.datetime(*time.strptime( |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
162 |
self.logfiles[0], "%Y%m%d%H")[0:6]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
163 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
164 |
def enough_history(self, ts): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
165 |
"""Returns true if the timestamp is so far behind the |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
166 |
update log, that there is not enough log history to bring |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
167 |
the client up to date.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
168 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
169 |
# Absence of server-side log history also counts as |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
170 |
# not enough history. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
171 |
if not self.last_update or not self.first_update: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
172 |
return False |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
173 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
174 |
if ts < self.first_update: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
175 |
return False |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
176 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
177 |
return True |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
178 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
179 |
@staticmethod |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
180 |
def recv(c, path, ts): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
181 |
"""Take a connection object and a catalog path. This method |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
182 |
receives a catalog from the server. If it is an incremental |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
183 |
update, it is processed by the updatelog. If it is a full |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
184 |
update, we call the catalog to handle the request. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
185 |
Ts is the timestamp when the local copy of the catalog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
186 |
was last modified.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
187 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
188 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
189 |
update_type = c.info().getheader("X-Catalog-Type", "full") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
190 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
191 |
if update_type == 'incremental': |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
192 |
UpdateLog._recv_updates(c, path, ts) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
193 |
else: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
194 |
catalog.recv(c, path) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
195 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
196 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
197 |
@staticmethod |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
198 |
def _recv_updates(filep, path, cts): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
199 |
"""A static method that takes a file-like object, |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
200 |
a path, and a timestamp. This is the other half of |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
201 |
send_updates(). It reads a stream as an incoming updatelog and |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
202 |
modifies the catalog on disk.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
203 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
204 |
if not os.path.exists(path): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
205 |
os.makedirs(path) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
206 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
207 |
# Build a list of FMRIs that this update would add, check to |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
208 |
# make sure that they aren't present in the catalog, then append |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
209 |
# the fmris. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
210 |
mts = catalog.ts_to_datetime(cts) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
211 |
cts = mts |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
212 |
added = 0 |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
213 |
npkgs = 0 |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
214 |
add_lines = [] |
220
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
215 |
unknown_lines = [] |
215
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
216 |
attrs = {} |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
217 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
218 |
for s in filep: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
219 |
l = s.split(None, 4) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
220 |
if len(l) < 4: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
221 |
continue |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
222 |
|
220
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
223 |
elif l[2] not in catalog.known_prefixes: |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
224 |
# Add unknown line directly to catalog. |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
225 |
# This can be post-processed later, when it |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
226 |
# becomes known. |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
227 |
# |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
228 |
# XXX Notify user that unknown entry was added? |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
229 |
ts = catalog.ts_to_datetime(l[1]) |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
230 |
if ts > cts: |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
231 |
if ts > mts: |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
232 |
mts = ts |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
233 |
str = "%s %s\n" % (l[2], l[3]) |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
234 |
unknown_lines.append(str) |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
235 |
|
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
236 |
elif l[0] == "+": |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
237 |
# This is a known entry type. |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
238 |
# Create a list of FMRIs to add, since |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
239 |
# additional inspection is required |
215
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
240 |
ts = catalog.ts_to_datetime(l[1]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
241 |
if ts > cts: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
242 |
if ts > mts: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
243 |
mts = ts |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
244 |
f = fmri.PkgFmri(l[3]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
245 |
str = "%s %s %s %s\n" % \ |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
246 |
(l[2], "pkg", f.pkg_name, f.version) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
247 |
add_lines.append(str) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
248 |
added += 1 |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
249 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
250 |
# Verify that they aren't already in the catalog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
251 |
catf = file(os.path.normpath( |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
252 |
os.path.join(path, "catalog")), "a+") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
253 |
catf.seek(0) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
254 |
for c in catf: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
255 |
npkgs += 1 |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
256 |
if c in add_lines: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
257 |
catf.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
258 |
raise UpdateLogException, \ |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
259 |
"Package %s is already in the catalog" % \ |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
260 |
c |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
261 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
262 |
# Write the new entries to the catalog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
263 |
catf.seek(0, 2) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
264 |
catf.writelines(add_lines) |
220
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
265 |
if len(unknown_lines) > 0: |
afceea12576c
320 pkg should ignore catalog lines it doesn't recognize
johansen <johansen@sun.com>
parents:
215
diff
changeset
|
266 |
catf.writelines(unknown_lines) |
215
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
267 |
catf.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
268 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
269 |
# Now re-write npkgs and Last-Modified in attributes file |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
270 |
afile = file(os.path.normpath(os.path.join(path, "attrs")), "r") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
271 |
attrre = re.compile('^S ([^:]*): (.*)') |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
272 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
273 |
for entry in afile: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
274 |
m = attrre.match(entry) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
275 |
if m != None: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
276 |
attrs[m.group(1)] = m.group(2) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
277 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
278 |
afile.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
279 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
280 |
# Update the attributes we care about |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
281 |
attrs["npkgs"] = npkgs + added |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
282 |
attrs["Last-Modified"] = mts.isoformat() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
283 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
284 |
# Write attributes back out |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
285 |
afile = file(os.path.normpath(os.path.join(path, "attrs")), "w") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
286 |
for a in attrs.keys(): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
287 |
s = "S %s: %s\n" % (a, attrs[a]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
288 |
afile.write(s) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
289 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
290 |
afile.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
291 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
292 |
return True |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
293 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
294 |
def send(self, request): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
295 |
"""This method takes a http request and sends a catalog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
296 |
to the client. If the client it capable of receiving an |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
297 |
incremental update, we'll send that. Otherwise, it calls |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
298 |
into the catalog to send a full copy.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
299 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
300 |
modified = request.headers.getheader("If-Modified-Since") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
301 |
ts = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
302 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
303 |
if modified: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
304 |
try: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
305 |
ts = catalog.ts_to_datetime(modified) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
306 |
except ValueError: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
307 |
ts = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
308 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
309 |
# Incremental catalog updates |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
310 |
if ts and self.up_to_date(ts): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
311 |
request.send_response(304) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
312 |
request.send_header('Content-type', 'text/plain') |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
313 |
request.send_header('Last-Modified', |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
314 |
self.catalog.last_modified()) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
315 |
request.send_header('X-Catalog-Type', 'incremental') |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
316 |
request.end_headers() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
317 |
return |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
318 |
elif ts and self.enough_history(ts): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
319 |
request.send_response(200) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
320 |
request.send_header('Content-type', 'text/plain') |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
321 |
request.send_header('Last-Modified', |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
322 |
self.catalog.last_modified()) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
323 |
request.send_header('X-Catalog-Type', 'incremental') |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
324 |
request.end_headers() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
325 |
self._send_updates(ts, request.wfile) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
326 |
return |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
327 |
else: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
328 |
# Not enough history, or full catalog requested |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
329 |
request.send_response(200) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
330 |
request.send_header('Content-type', 'text/plain') |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
331 |
request.send_header('X-Catalog-Type', 'full') |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
332 |
request.end_headers() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
333 |
self.catalog.send(request.wfile) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
334 |
return |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
335 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
336 |
def _send_updates(self, ts, filep): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
337 |
"""Look through the logs for updates that have occurred |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
338 |
after timestamp. Write these changes to the file-like object |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
339 |
supplied in filep.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
340 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
341 |
# The files that need to be examined depend upon the timestamp |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
342 |
# supplied by the client, and the log files actually present. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
343 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
344 |
# The following cases exist: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
345 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
346 |
# 1. No updates have occurred since timestamp. Send nothing. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
347 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
348 |
# 2. Timestamp is older than oldest log record. Client needs to |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
349 |
# download full catalog. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
350 |
# |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
351 |
# 3. Timestamp falls within a range for which update records |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
352 |
# exist. If the timestamp is in the middle of a log-file, open |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
353 |
# that file, send updates newer than timestamp, and then send |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
354 |
# all newer files. Otherwise, just send updates from the newer |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
355 |
# log files. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
356 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
357 |
if self.up_to_date(ts) or not self.enough_history(ts): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
358 |
return |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
359 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
360 |
# Remove minutes, seconds, and microsec from timestamp |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
361 |
rts = datetime.datetime(ts.year, ts.month, ts.day, ts.hour) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
362 |
assert rts < ts |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
363 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
364 |
# send data from logfiles newer or equal to rts |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
365 |
for lf in self.logfiles: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
366 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
367 |
lf_time = datetime.datetime( |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
368 |
*time.strptime(lf, "%Y%m%d%H")[0:6]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
369 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
370 |
if lf_time >= rts: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
371 |
fn = "%s" % lf |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
372 |
logf = file(os.path.join(self.rootdir, fn), "r") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
373 |
for line in logf: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
374 |
filep.write(line) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
375 |
logf.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
376 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
377 |
def _setup_logfiles(self): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
378 |
"""Scans the directory containing the update log's files. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
379 |
Sets up any necessary state for the UpdateLog.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
380 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
381 |
# Store names of logfiles as integers for easier comparison |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
382 |
self.logfiles = [f for f in os.listdir(self.rootdir)] |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
383 |
self.logfiles.sort() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
384 |
self.curfiles = len(self.logfiles) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
385 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
386 |
if self.curfiles == 0: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
387 |
self.last_update = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
388 |
self.first_update = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
389 |
return |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
390 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
391 |
# Find the last update by opening the most recent logfile |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
392 |
# and finding its last entry |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
393 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
394 |
filenm = self.logfiles[self.curfiles - 1] |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
395 |
logf = file(os.path.join(self.rootdir, filenm), "r") |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
396 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
397 |
last_update = None |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
398 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
399 |
for ln in logf: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
400 |
lspl = ln.split(" ", 4) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
401 |
if len(lspl) < 4: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
402 |
continue |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
403 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
404 |
current_ts = catalog.ts_to_datetime(lspl[1]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
405 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
406 |
if not last_update or current_ts > last_update: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
407 |
last_update = current_ts |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
408 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
409 |
logf.close() |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
410 |
self.last_update = last_update |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
411 |
self.first_update = datetime.datetime( |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
412 |
*time.strptime(self.logfiles[0], "%Y%m%d%H")[0:6]) |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
413 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
414 |
def up_to_date(self, ts): |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
415 |
"""Returns true if the timestamp is up to date.""" |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
416 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
417 |
if self.last_update and ts >= self.last_update: |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
418 |
return True |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
419 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
420 |
return False |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
421 |
|
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
422 |
# Allow these methods to be invoked without explictly naming the UpdateLog |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
423 |
# class. |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
424 |
recv = UpdateLog.recv |
c10719939c6d
245 Need incremental update mechanism for the catalog
johansen <johansen@sun.com>
parents:
diff
changeset
|
425 |