Index: src/attach.c
==================================================================
--- src/attach.c
+++ src/attach.c
@@ -241,11 +241,11 @@
int rid;
int i, n;
db_begin_transaction();
blob_init(&content, aContent, szContent);
- rid = content_put(&content, 0, 0);
+ rid = content_put(&content, 0, 0, 0);
zUUID = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
blob_zero(&manifest);
for(i=n=0; zName[i]; i++){
if( zName[i]=='/' || zName[i]=='\\' ) n = i;
}
@@ -263,11 +263,11 @@
zDate[10] = 'T';
blob_appendf(&manifest, "D %s\n", zDate);
blob_appendf(&manifest, "U %F\n", g.zLogin ? g.zLogin : "nobody");
md5sum_blob(&manifest, &cksum);
blob_appendf(&manifest, "Z %b\n", &cksum);
- rid = content_put(&manifest, 0, 0);
+ rid = content_put(&manifest, 0, 0, 0);
manifest_crosslink(rid, &manifest);
db_end_transaction(0);
cgi_redirect(zFrom);
}
style_header("Add Attachment");
@@ -343,11 +343,11 @@
zDate[10] = 'T';
blob_appendf(&manifest, "D %s\n", zDate);
blob_appendf(&manifest, "U %F\n", g.zLogin ? g.zLogin : "nobody");
md5sum_blob(&manifest, &cksum);
blob_appendf(&manifest, "Z %b\n", &cksum);
- rid = content_put(&manifest, 0, 0);
+ rid = content_put(&manifest, 0, 0, 0);
manifest_crosslink(rid, &manifest);
db_end_transaction(0);
cgi_redirect(zFrom);
}
style_header("Delete Attachment");
Index: src/branch.c
==================================================================
--- src/branch.c
+++ src/branch.c
@@ -137,11 +137,11 @@
db_end_transaction(1);
fossil_exit(1);
}
}
- brid = content_put(&branch, 0, 0);
+ brid = content_put(&branch, 0, 0, 0);
if( brid==0 ){
fossil_panic("trouble committing manifest: %s", g.zErrMsg);
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", brid);
if( manifest_crosslink(brid, &branch)==0 ){
Index: src/checkin.c
==================================================================
--- src/checkin.c
+++ src/checkin.c
@@ -959,11 +959,11 @@
zFullname = db_column_text(&q, 1);
rid = db_column_int(&q, 2);
blob_zero(&content);
blob_read_from_file(&content, zFullname);
- nrid = content_put(&content, 0, 0);
+ nrid = content_put(&content, 0, 0, 0);
blob_reset(&content);
if( rid>0 ){
content_deltify(rid, nrid, 0);
}
db_multi_exec("UPDATE vfile SET mrid=%d, rid=%d WHERE id=%d", nrid,nrid,id);
@@ -1051,11 +1051,11 @@
blob_write_to_file(&manifest, zManifestFile);
blob_reset(&manifest);
blob_read_from_file(&manifest, zManifestFile);
free(zManifestFile);
}
- nvid = content_put(&manifest, 0, 0);
+ nvid = content_put(&manifest, 0, 0, 0);
if( nvid==0 ){
fossil_panic("trouble committing manifest: %s", g.zErrMsg);
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nvid);
manifest_crosslink(nvid, &manifest);
Index: src/content.c
==================================================================
--- src/content.c
+++ src/content.c
@@ -421,20 +421,22 @@
/*
** Write content into the database. Return the record ID. If the
** content is already in the database, just return the record ID.
**
** If srcId is specified, then pBlob is delta content from
-** the srcId record. srcId might be a phantom.
+** the srcId record. srcId might be a phantom. If nBlob>0 then the
+** pBlob value has already been compressed and nBlob is its uncompressed
+** size. If nBlob>0 then zUuid must be valid.
**
** zUuid is the UUID of the artifact, if it is specified. When srcId is
** specified then zUuid must always be specified. If srcId is zero,
** and zUuid is zero then the correct zUuid is computed from pBlob.
**
** If the record already exists but is a phantom, the pBlob content
** is inserted and the phatom becomes a real record.
*/
-int content_put(Blob *pBlob, const char *zUuid, int srcId){
+int content_put(Blob *pBlob, const char *zUuid, int srcId, int nBlob){
int size;
int rid;
Stmt s1;
Blob cmpr;
Blob hash;
@@ -444,15 +446,16 @@
assert( g.repositoryOpen );
assert( pBlob!=0 );
assert( srcId==0 || zUuid!=0 );
if( zUuid==0 ){
assert( pBlob!=0 );
+ assert( nBlob==0 );
sha1sum_blob(pBlob, &hash);
}else{
blob_init(&hash, zUuid, -1);
}
- size = blob_size(pBlob);
+ size = nBlob ? nBlob : blob_size(pBlob);
db_begin_transaction();
/* Check to see if the entry already exists and if it does whether
** or not the entry is a phantom
*/
@@ -481,11 +484,15 @@
g.userUid, g.zNonce, g.zIpAddr
);
g.rcvid = db_last_insert_rowid();
}
- blob_compress(pBlob, &cmpr);
+ if( nBlob ){
+ cmpr = pBlob[0];
+ }else{
+ blob_compress(pBlob, &cmpr);
+ }
if( rid>0 ){
/* We are just adding data to a phantom */
db_prepare(&s1,
"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
g.rcvid, size, rid
@@ -513,11 +520,11 @@
if( g.markPrivate ){
db_multi_exec("INSERT INTO private VALUES(%d)", rid);
markAsUnclustered = 0;
}
}
- blob_reset(&cmpr);
+ if( nBlob==0 ) blob_reset(&cmpr);
/* If the srcId is specified, then the data we just added is
** really a delta. Record this fact in the delta table.
*/
if( srcId ){
@@ -590,20 +597,20 @@
/*
** COMMAND: test-content-put
**
-** Extract a blob from the database and write it into a file.
+** Extract a blob from a file and write it into the database
*/
void test_content_put_cmd(void){
int rid;
Blob content;
if( g.argc!=3 ) usage("FILENAME");
db_must_be_within_tree();
user_select();
blob_read_from_file(&content, g.argv[2]);
- rid = content_put(&content, 0, 0);
+ rid = content_put(&content, 0, 0, 0);
printf("inserted as record %d\n", rid);
}
/*
** Make sure the content at rid is the original content and is not a
Index: src/db.c
==================================================================
--- src/db.c
+++ src/db.c
@@ -357,10 +357,13 @@
return sqlite3_column_double(pStmt->pStmt, N);
}
const char *db_column_text(Stmt *pStmt, int N){
return (char*)sqlite3_column_text(pStmt->pStmt, N);
}
+const char *db_column_raw(Stmt *pStmt, int N){
+ return (const char*)sqlite3_column_blob(pStmt->pStmt, N);
+}
const char *db_column_name(Stmt *pStmt, int N){
return (char*)sqlite3_column_name(pStmt->pStmt, N);
}
int db_column_count(Stmt *pStmt){
return sqlite3_column_count(pStmt->pStmt);
@@ -1053,11 +1056,11 @@
blob_appendf(&manifest, "T *sym-trunk *\n");
blob_appendf(&manifest, "U %F\n", g.zLogin);
md5sum_blob(&manifest, &hash);
blob_appendf(&manifest, "Z %b\n", &hash);
blob_reset(&hash);
- rid = content_put(&manifest, 0, 0);
+ rid = content_put(&manifest, 0, 0, 0);
manifest_crosslink(rid, &manifest);
}
}
/*
Index: src/event.c
==================================================================
--- src/event.c
+++ src/event.c
@@ -344,11 +344,11 @@
}
blob_appendf(&event, "W %d\n%s\n", strlen(zBody), zBody);
md5sum_blob(&event, &cksum);
blob_appendf(&event, "Z %b\n", &cksum);
blob_reset(&cksum);
- nrid = content_put(&event, 0, 0);
+ nrid = content_put(&event, 0, 0, 0);
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
manifest_crosslink(nrid, &event);
blob_reset(&event);
content_deltify(rid, nrid, 0);
db_end_transaction(0);
Index: src/info.c
==================================================================
--- src/info.c
+++ src/info.c
@@ -1521,11 +1521,11 @@
blob_appendf(&ctrl, "U %F\n", g.zLogin);
md5sum_blob(&ctrl, &cksum);
blob_appendf(&ctrl, "Z %b\n", &cksum);
db_begin_transaction();
g.markPrivate = content_is_private(rid);
- nrid = content_put(&ctrl, 0, 0);
+ nrid = content_put(&ctrl, 0, 0, 0);
manifest_crosslink(nrid, &ctrl);
db_end_transaction(0);
}
cgi_redirectf("ci?name=%s", zUuid);
}
Index: src/rebuild.c
==================================================================
--- src/rebuild.c
+++ src/rebuild.c
@@ -518,11 +518,11 @@
blob_appendf(&path, "%s", zSubpath);
if( blob_read_from_file(&aContent, blob_str(&path))==-1 ){
fossil_panic("some unknown error occurred while reading \"%s\"",
blob_str(&path));
}
- content_put(&aContent, 0, 0);
+ content_put(&aContent, 0, 0, 0);
blob_reset(&path);
blob_reset(&aContent);
free(zSubpath);
printf("\r%d", ++nFileRead);
fflush(stdout);
Index: src/tag.c
==================================================================
--- src/tag.c
+++ src/tag.c
@@ -307,11 +307,11 @@
blob_appendf(&ctrl, "\n");
}
blob_appendf(&ctrl, "U %F\n", zUserOvrd ? zUserOvrd : g.zLogin);
md5sum_blob(&ctrl, &cksum);
blob_appendf(&ctrl, "Z %b\n", &cksum);
- nrid = content_put(&ctrl, 0, 0);
+ nrid = content_put(&ctrl, 0, 0, 0);
manifest_crosslink(nrid, &ctrl);
}
/*
** COMMAND: tag
Index: src/tkt.c
==================================================================
--- src/tkt.c
+++ src/tkt.c
@@ -480,11 +480,11 @@
}else if( g.thTrace ){
Th_Trace("submit_ticket {\n
\n%h\n
\n"
"}
\n",
blob_str(&tktchng));
}else{
- rid = content_put(&tktchng, 0, 0);
+ rid = content_put(&tktchng, 0, 0, 0);
if( rid==0 ){
fossil_panic("trouble committing ticket: %s", g.zErrMsg);
}
manifest_crosslink_begin();
manifest_crosslink(rid, &tktchng);
@@ -1056,11 +1056,11 @@
}
blob_appendf(&tktchng, "K %s\n", zTktUuid);
blob_appendf(&tktchng, "U %F\n", g.zLogin);
md5sum_blob(&tktchng, &cksum);
blob_appendf(&tktchng, "Z %b\n", &cksum);
- rid = content_put(&tktchng, 0, 0);
+ rid = content_put(&tktchng, 0, 0, 0);
if( rid==0 ){
fossil_panic("trouble committing ticket: %s", g.zErrMsg);
}
manifest_crosslink_begin();
manifest_crosslink(rid, &tktchng);
Index: src/wiki.c
==================================================================
--- src/wiki.c
+++ src/wiki.c
@@ -322,11 +322,11 @@
}
blob_appendf(&wiki, "W %d\n%s\n", strlen(zBody), zBody);
md5sum_blob(&wiki, &cksum);
blob_appendf(&wiki, "Z %b\n", &cksum);
blob_reset(&cksum);
- nrid = content_put(&wiki, 0, 0);
+ nrid = content_put(&wiki, 0, 0, 0);
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
manifest_crosslink(nrid, &wiki);
blob_reset(&wiki);
content_deltify(rid, nrid, 0);
}
@@ -495,11 +495,11 @@
appendRemark(&body);
blob_appendf(&wiki, "W %d\n%s\n", blob_size(&body), blob_str(&body));
md5sum_blob(&wiki, &cksum);
blob_appendf(&wiki, "Z %b\n", &cksum);
blob_reset(&cksum);
- nrid = content_put(&wiki, 0, 0);
+ nrid = content_put(&wiki, 0, 0, 0);
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
manifest_crosslink(nrid, &wiki);
blob_reset(&wiki);
content_deltify(rid, nrid, 0);
db_end_transaction(0);
@@ -820,11 +820,11 @@
blob_str(pContent) );
md5sum_blob(&wiki, &cksum);
blob_appendf(&wiki, "Z %b\n", &cksum);
blob_reset(&cksum);
db_begin_transaction();
- nrid = content_put( &wiki, 0, 0 );
+ nrid = content_put( &wiki, 0, 0, 0);
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
manifest_crosslink(nrid,&wiki);
blob_reset(&wiki);
content_deltify(rid,nrid,0);
db_end_transaction(0);
Index: src/xfer.c
==================================================================
--- src/xfer.c
+++ src/xfer.c
@@ -27,11 +27,11 @@
typedef struct Xfer Xfer;
struct Xfer {
Blob *pIn; /* Input text from the other side */
Blob *pOut; /* Compose our reply here */
Blob line; /* The current line of input */
- Blob aToken[5]; /* Tokenized version of line */
+ Blob aToken[6]; /* Tokenized version of line */
Blob err; /* Error message text */
int nToken; /* Number of tokens in line */
int nIGotSent; /* Number of "igot" cards sent */
int nGimmeSent; /* Number of gimme cards sent */
int nFileSent; /* Number of files sent */
@@ -132,20 +132,20 @@
pXfer->nDeltaRcvd++;
}else{
srcid = 0;
pXfer->nFileRcvd++;
}
- rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
+ rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid, 0);
remote_has(rid);
blob_reset(&content);
return;
}
if( pXfer->nToken==4 ){
Blob src, next;
srcid = rid_from_uuid(&pXfer->aToken[2], 1);
if( content_get(srcid, &src)==0 ){
- rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
+ rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid, 0);
pXfer->nDanglingFile++;
db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
content_make_public(rid);
return;
}
@@ -159,20 +159,79 @@
}
sha1sum_blob(&content, &hash);
if( !blob_eq_str(&pXfer->aToken[1], blob_str(&hash), -1) ){
blob_appendf(&pXfer->err, "content does not match sha1 hash");
}
- rid = content_put(&content, blob_str(&hash), 0);
+ rid = content_put(&content, blob_str(&hash), 0, 0);
blob_reset(&hash);
if( rid==0 ){
blob_appendf(&pXfer->err, "%s", g.zErrMsg);
}else{
content_make_public(rid);
manifest_crosslink(rid, &content);
}
remote_has(rid);
}
+
+/*
+** The aToken[0..nToken-1] blob array is a parse of a "cfile" line
+** message. This routine finishes parsing that message and does
+** a record insert of the file. The difference between "file" and
+** "cfile" is that with "cfile" the content is already compressed.
+**
+** The file line is in one of the following two forms:
+**
+** cfile UUID USIZE CSIZE \n CONTENT
+** cfile UUID DELTASRC USIZE CSIZE \n CONTENT
+**
+** The content is CSIZE bytes immediately following the newline.
+** If DELTASRC exists, then the CONTENT is a delta against the
+** content of DELTASRC.
+**
+** The original size of the UUID artifact is USIZE.
+**
+** If any error occurs, write a message into pErr which has already
+** be initialized to an empty string.
+**
+** Any artifact successfully received by this routine is considered to
+** be public and is therefore removed from the "private" table.
+*/
+static void xfer_accept_compressed_file(Xfer *pXfer){
+ int szC; /* CSIZE */
+ int szU; /* USIZE */
+ int rid;
+ int srcid = 0;
+ Blob content;
+
+ if( pXfer->nToken<4
+ || pXfer->nToken>5
+ || !blob_is_uuid(&pXfer->aToken[1])
+ || !blob_is_int(&pXfer->aToken[pXfer->nToken-2], &szU)
+ || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &szC)
+ || szC<0 || szU<0
+ || (pXfer->nToken==5 && !blob_is_uuid(&pXfer->aToken[2]))
+ ){
+ blob_appendf(&pXfer->err, "malformed cfile line");
+ return;
+ }
+ blob_zero(&content);
+ blob_extract(pXfer->pIn, szC, &content);
+ if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
+ /* Ignore files that have been shunned */
+ return;
+ }
+ if( pXfer->nToken==5 ){
+ srcid = rid_from_uuid(&pXfer->aToken[2], 1);
+ pXfer->nDeltaRcvd++;
+ }else{
+ srcid = 0;
+ pXfer->nFileRcvd++;
+ }
+ rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid, szC);
+ remote_has(rid);
+ blob_reset(&content);
+}
/*
** Try to send a file as a delta against its parent.
** If successful, return the number of bytes in the delta.
** If we cannot generate an appropriate delta, then send
@@ -333,10 +392,55 @@
}
}
remote_has(rid);
blob_reset(&uuid);
}
+
+/*
+** Send the file identified by rid as a compressed artifact. Basically,
+** send the content exactly as it appears in the BLOB table using
+** a "cfile" card.
+*/
+static void send_compressed_file(Xfer *pXfer, int rid){
+ const char *zContent;
+ const char *zUuid;
+ char *zDelta;
+ int szU;
+ int szC;
+ int rc;
+ Stmt s;
+
+ db_prepare(&s,
+ "SELECT uuid, size, content FROM blob"
+ " WHERE rid=%d"
+ " AND size>=0"
+ " AND uuid NOT IN shun"
+ " AND rid NOT IN private",
+ rid
+ );
+ rc = db_step(&s);
+ if( rc==SQLITE_ROW ){
+ zUuid = db_column_text(&s, 0);
+ szU = db_column_int(&s, 1);
+ szC = db_column_bytes(&s, 2);
+ zContent = db_column_raw(&s, 2);
+ zDelta = db_text(0, "SELECT uuid FROM blob WHERE rid="
+ " (SELECT srcid FROM delta WHERE rid=%d)", rid);
+ blob_appendf(pXfer->pOut, "cfile %s ", zUuid);
+ if( zDelta ){
+ blob_appendf(pXfer->pOut, "%s ", zDelta);
+ fossil_free(zDelta);
+ pXfer->nDeltaSent++;
+ }else{
+ pXfer->nFileSent++;
+ }
+ blob_appendf(pXfer->pOut, "%d %d\n", szU, szC);
+ blob_append(pXfer->pOut, zContent, szC);
+ blob_append(pXfer->pOut, "\n", 1);
+ }
+ db_finalize(&s);
+}
/*
** Send a gimme message for every phantom.
**
** It should not be possible to have a private phantom. But just to be
@@ -511,11 +615,11 @@
nRow++;
if( nRow>=800 && nUncl>nRow+100 ){
md5sum_blob(&cluster, &cksum);
blob_appendf(&cluster, "Z %b\n", &cksum);
blob_reset(&cksum);
- content_put(&cluster, 0, 0);
+ content_put(&cluster, 0, 0, 0);
blob_reset(&cluster);
nUncl -= nRow;
nRow = 0;
}
}
@@ -523,11 +627,11 @@
db_multi_exec("DELETE FROM unclustered");
if( nRow>0 ){
md5sum_blob(&cluster, &cksum);
blob_appendf(&cluster, "Z %b\n", &cksum);
blob_reset(&cksum);
- content_put(&cluster, 0, 0);
+ content_put(&cluster, 0, 0, 0);
blob_reset(&cluster);
}
}
}
@@ -666,10 +770,31 @@
@ error %T(blob_str(&xfer.err))
nErr++;
break;
}
}else
+
+ /* cfile UUID USIZE CSIZE \n CONTENT
+ ** cfile UUID DELTASRC USIZE CSIZE \n CONTENT
+ **
+ ** Accept a file from the client.
+ */
+ if( blob_eq(&xfer.aToken[0], "cfile") ){
+ if( !isPush ){
+ cgi_reset_content();
+ @ error not\sauthorized\sto\swrite
+ nErr++;
+ break;
+ }
+ xfer_accept_compressed_file(&xfer);
+ if( blob_size(&xfer.err) ){
+ cgi_reset_content();
+ @ error %T(blob_str(&xfer.err))
+ nErr++;
+ break;
+ }
+ }else
/* gimme UUID
**
** Client is requesting a file. Send it.
*/
@@ -763,14 +888,21 @@
if( xfer.nToken==3
&& blob_is_int(&xfer.aToken[1], &iVers)
&& iVers>=2
){
int seqno, max;
+ if( iVers>=3 ){
+ cgi_set_content_type("application/x-fossil-uncompressed");
+ }
blob_is_int(&xfer.aToken[2], &seqno);
max = db_int(0, "SELECT max(rid) FROM blob");
while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){
- send_file(&xfer, seqno, 0, 1);
+ if( iVers>=3 ){
+ send_compressed_file(&xfer, seqno);
+ }else{
+ send_file(&xfer, seqno, 0, 1);
+ }
seqno++;
}
if( seqno>=max ) seqno = 0;
@ clone_seqno %d(seqno)
}else{
@@ -1032,11 +1164,11 @@
/*
** Always begin with a clone, pull, or push message
*/
if( cloneFlag ){
- blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
+ blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
pushFlag = 0;
pullFlag = 0;
nCardSent++;
/* TBD: Request all transferable configuration values */
content_enable_dephantomize(0);
@@ -1191,10 +1323,19 @@
** Receive a file transmitted from the server.
*/
if( blob_eq(&xfer.aToken[0],"file") ){
xfer_accept_file(&xfer, cloneFlag);
}else
+
+ /* cfile UUID USIZE CSIZE \n CONTENT
+ ** cfile UUID DELTASRC USIZE CSIZE \n CONTENT
+ **
+ ** Receive a compressed file transmitted from the server.
+ */
+ if( blob_eq(&xfer.aToken[0],"cfile") ){
+ xfer_accept_compressed_file(&xfer);
+ }else
/* gimme UUID
**
** Server is requesting a file. If the file is a manifest, assume
** that the server will also want to know all of the content files
@@ -1249,11 +1390,11 @@
}
if( zPCode==0 ){
zPCode = mprintf("%b", &xfer.aToken[2]);
db_set("project-code", zPCode, 0);
}
- blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
+ blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
nCardSent++;
}else
/* config NAME SIZE \n CONTENT
**
@@ -1363,20 +1504,20 @@
break;
}
}else
/* Unknown message */
- {
+ if( xfer.nToken>0 ){
if( blob_str(&xfer.aToken[0])[0]=='<' ){
fossil_warning(
"server replies with HTML instead of fossil sync protocol:\n%b",
&recv
);
nErr++;
break;
}
- blob_appendf(&xfer.err, "unknown command: %b", &xfer.aToken[0]);
+ blob_appendf(&xfer.err, "unknown command: [%b]", &xfer.aToken[0]);
}
if( blob_size(&xfer.err) ){
fossil_warning("%b", &xfer.err);
nErr++;