Index: src/attach.c ================================================================== --- src/attach.c +++ src/attach.c @@ -241,11 +241,11 @@ int rid; int i, n; db_begin_transaction(); blob_init(&content, aContent, szContent); - rid = content_put(&content, 0, 0); + rid = content_put(&content, 0, 0, 0); zUUID = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid); blob_zero(&manifest); for(i=n=0; zName[i]; i++){ if( zName[i]=='/' || zName[i]=='\\' ) n = i; } @@ -263,11 +263,11 @@ zDate[10] = 'T'; blob_appendf(&manifest, "D %s\n", zDate); blob_appendf(&manifest, "U %F\n", g.zLogin ? g.zLogin : "nobody"); md5sum_blob(&manifest, &cksum); blob_appendf(&manifest, "Z %b\n", &cksum); - rid = content_put(&manifest, 0, 0); + rid = content_put(&manifest, 0, 0, 0); manifest_crosslink(rid, &manifest); db_end_transaction(0); cgi_redirect(zFrom); } style_header("Add Attachment"); @@ -343,11 +343,11 @@ zDate[10] = 'T'; blob_appendf(&manifest, "D %s\n", zDate); blob_appendf(&manifest, "U %F\n", g.zLogin ? g.zLogin : "nobody"); md5sum_blob(&manifest, &cksum); blob_appendf(&manifest, "Z %b\n", &cksum); - rid = content_put(&manifest, 0, 0); + rid = content_put(&manifest, 0, 0, 0); manifest_crosslink(rid, &manifest); db_end_transaction(0); cgi_redirect(zFrom); } style_header("Delete Attachment"); Index: src/branch.c ================================================================== --- src/branch.c +++ src/branch.c @@ -137,11 +137,11 @@ db_end_transaction(1); fossil_exit(1); } } - brid = content_put(&branch, 0, 0); + brid = content_put(&branch, 0, 0, 0); if( brid==0 ){ fossil_panic("trouble committing manifest: %s", g.zErrMsg); } db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", brid); if( manifest_crosslink(brid, &branch)==0 ){ Index: src/checkin.c ================================================================== --- src/checkin.c +++ src/checkin.c @@ -959,11 +959,11 @@ zFullname = db_column_text(&q, 1); rid = db_column_int(&q, 2); blob_zero(&content); blob_read_from_file(&content, zFullname); - nrid = content_put(&content, 0, 0); + nrid = content_put(&content, 0, 0, 0); blob_reset(&content); if( rid>0 ){ content_deltify(rid, nrid, 0); } db_multi_exec("UPDATE vfile SET mrid=%d, rid=%d WHERE id=%d", nrid,nrid,id); @@ -1051,11 +1051,11 @@ blob_write_to_file(&manifest, zManifestFile); blob_reset(&manifest); blob_read_from_file(&manifest, zManifestFile); free(zManifestFile); } - nvid = content_put(&manifest, 0, 0); + nvid = content_put(&manifest, 0, 0, 0); if( nvid==0 ){ fossil_panic("trouble committing manifest: %s", g.zErrMsg); } db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nvid); manifest_crosslink(nvid, &manifest); Index: src/content.c ================================================================== --- src/content.c +++ src/content.c @@ -421,20 +421,22 @@ /* ** Write content into the database. Return the record ID. If the ** content is already in the database, just return the record ID. ** ** If srcId is specified, then pBlob is delta content from -** the srcId record. srcId might be a phantom. +** the srcId record. srcId might be a phantom. If nBlob>0 then the +** pBlob value has already been compressed and nBlob is its uncompressed +** size. If nBlob>0 then zUuid must be valid. ** ** zUuid is the UUID of the artifact, if it is specified. When srcId is ** specified then zUuid must always be specified. If srcId is zero, ** and zUuid is zero then the correct zUuid is computed from pBlob. ** ** If the record already exists but is a phantom, the pBlob content ** is inserted and the phatom becomes a real record. */ -int content_put(Blob *pBlob, const char *zUuid, int srcId){ +int content_put(Blob *pBlob, const char *zUuid, int srcId, int nBlob){ int size; int rid; Stmt s1; Blob cmpr; Blob hash; @@ -444,15 +446,16 @@ assert( g.repositoryOpen ); assert( pBlob!=0 ); assert( srcId==0 || zUuid!=0 ); if( zUuid==0 ){ assert( pBlob!=0 ); + assert( nBlob==0 ); sha1sum_blob(pBlob, &hash); }else{ blob_init(&hash, zUuid, -1); } - size = blob_size(pBlob); + size = nBlob ? nBlob : blob_size(pBlob); db_begin_transaction(); /* Check to see if the entry already exists and if it does whether ** or not the entry is a phantom */ @@ -481,11 +484,15 @@ g.userUid, g.zNonce, g.zIpAddr ); g.rcvid = db_last_insert_rowid(); } - blob_compress(pBlob, &cmpr); + if( nBlob ){ + cmpr = pBlob[0]; + }else{ + blob_compress(pBlob, &cmpr); + } if( rid>0 ){ /* We are just adding data to a phantom */ db_prepare(&s1, "UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d", g.rcvid, size, rid @@ -513,11 +520,11 @@ if( g.markPrivate ){ db_multi_exec("INSERT INTO private VALUES(%d)", rid); markAsUnclustered = 0; } } - blob_reset(&cmpr); + if( nBlob==0 ) blob_reset(&cmpr); /* If the srcId is specified, then the data we just added is ** really a delta. Record this fact in the delta table. */ if( srcId ){ @@ -590,20 +597,20 @@ /* ** COMMAND: test-content-put ** -** Extract a blob from the database and write it into a file. +** Extract a blob from a file and write it into the database */ void test_content_put_cmd(void){ int rid; Blob content; if( g.argc!=3 ) usage("FILENAME"); db_must_be_within_tree(); user_select(); blob_read_from_file(&content, g.argv[2]); - rid = content_put(&content, 0, 0); + rid = content_put(&content, 0, 0, 0); printf("inserted as record %d\n", rid); } /* ** Make sure the content at rid is the original content and is not a Index: src/db.c ================================================================== --- src/db.c +++ src/db.c @@ -357,10 +357,13 @@ return sqlite3_column_double(pStmt->pStmt, N); } const char *db_column_text(Stmt *pStmt, int N){ return (char*)sqlite3_column_text(pStmt->pStmt, N); } +const char *db_column_raw(Stmt *pStmt, int N){ + return (const char*)sqlite3_column_blob(pStmt->pStmt, N); +} const char *db_column_name(Stmt *pStmt, int N){ return (char*)sqlite3_column_name(pStmt->pStmt, N); } int db_column_count(Stmt *pStmt){ return sqlite3_column_count(pStmt->pStmt); @@ -1053,11 +1056,11 @@ blob_appendf(&manifest, "T *sym-trunk *\n"); blob_appendf(&manifest, "U %F\n", g.zLogin); md5sum_blob(&manifest, &hash); blob_appendf(&manifest, "Z %b\n", &hash); blob_reset(&hash); - rid = content_put(&manifest, 0, 0); + rid = content_put(&manifest, 0, 0, 0); manifest_crosslink(rid, &manifest); } } /* Index: src/event.c ================================================================== --- src/event.c +++ src/event.c @@ -344,11 +344,11 @@ } blob_appendf(&event, "W %d\n%s\n", strlen(zBody), zBody); md5sum_blob(&event, &cksum); blob_appendf(&event, "Z %b\n", &cksum); blob_reset(&cksum); - nrid = content_put(&event, 0, 0); + nrid = content_put(&event, 0, 0, 0); db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid); manifest_crosslink(nrid, &event); blob_reset(&event); content_deltify(rid, nrid, 0); db_end_transaction(0); Index: src/info.c ================================================================== --- src/info.c +++ src/info.c @@ -1521,11 +1521,11 @@ blob_appendf(&ctrl, "U %F\n", g.zLogin); md5sum_blob(&ctrl, &cksum); blob_appendf(&ctrl, "Z %b\n", &cksum); db_begin_transaction(); g.markPrivate = content_is_private(rid); - nrid = content_put(&ctrl, 0, 0); + nrid = content_put(&ctrl, 0, 0, 0); manifest_crosslink(nrid, &ctrl); db_end_transaction(0); } cgi_redirectf("ci?name=%s", zUuid); } Index: src/rebuild.c ================================================================== --- src/rebuild.c +++ src/rebuild.c @@ -518,11 +518,11 @@ blob_appendf(&path, "%s", zSubpath); if( blob_read_from_file(&aContent, blob_str(&path))==-1 ){ fossil_panic("some unknown error occurred while reading \"%s\"", blob_str(&path)); } - content_put(&aContent, 0, 0); + content_put(&aContent, 0, 0, 0); blob_reset(&path); blob_reset(&aContent); free(zSubpath); printf("\r%d", ++nFileRead); fflush(stdout); Index: src/tag.c ================================================================== --- src/tag.c +++ src/tag.c @@ -307,11 +307,11 @@ blob_appendf(&ctrl, "\n"); } blob_appendf(&ctrl, "U %F\n", zUserOvrd ? zUserOvrd : g.zLogin); md5sum_blob(&ctrl, &cksum); blob_appendf(&ctrl, "Z %b\n", &cksum); - nrid = content_put(&ctrl, 0, 0); + nrid = content_put(&ctrl, 0, 0, 0); manifest_crosslink(nrid, &ctrl); } /* ** COMMAND: tag Index: src/tkt.c ================================================================== --- src/tkt.c +++ src/tkt.c @@ -480,11 +480,11 @@ }else if( g.thTrace ){ Th_Trace("submit_ticket {\n
\n%h\n
\n" "}
\n", blob_str(&tktchng)); }else{ - rid = content_put(&tktchng, 0, 0); + rid = content_put(&tktchng, 0, 0, 0); if( rid==0 ){ fossil_panic("trouble committing ticket: %s", g.zErrMsg); } manifest_crosslink_begin(); manifest_crosslink(rid, &tktchng); @@ -1056,11 +1056,11 @@ } blob_appendf(&tktchng, "K %s\n", zTktUuid); blob_appendf(&tktchng, "U %F\n", g.zLogin); md5sum_blob(&tktchng, &cksum); blob_appendf(&tktchng, "Z %b\n", &cksum); - rid = content_put(&tktchng, 0, 0); + rid = content_put(&tktchng, 0, 0, 0); if( rid==0 ){ fossil_panic("trouble committing ticket: %s", g.zErrMsg); } manifest_crosslink_begin(); manifest_crosslink(rid, &tktchng); Index: src/wiki.c ================================================================== --- src/wiki.c +++ src/wiki.c @@ -322,11 +322,11 @@ } blob_appendf(&wiki, "W %d\n%s\n", strlen(zBody), zBody); md5sum_blob(&wiki, &cksum); blob_appendf(&wiki, "Z %b\n", &cksum); blob_reset(&cksum); - nrid = content_put(&wiki, 0, 0); + nrid = content_put(&wiki, 0, 0, 0); db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid); manifest_crosslink(nrid, &wiki); blob_reset(&wiki); content_deltify(rid, nrid, 0); } @@ -495,11 +495,11 @@ appendRemark(&body); blob_appendf(&wiki, "W %d\n%s\n", blob_size(&body), blob_str(&body)); md5sum_blob(&wiki, &cksum); blob_appendf(&wiki, "Z %b\n", &cksum); blob_reset(&cksum); - nrid = content_put(&wiki, 0, 0); + nrid = content_put(&wiki, 0, 0, 0); db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid); manifest_crosslink(nrid, &wiki); blob_reset(&wiki); content_deltify(rid, nrid, 0); db_end_transaction(0); @@ -820,11 +820,11 @@ blob_str(pContent) ); md5sum_blob(&wiki, &cksum); blob_appendf(&wiki, "Z %b\n", &cksum); blob_reset(&cksum); db_begin_transaction(); - nrid = content_put( &wiki, 0, 0 ); + nrid = content_put( &wiki, 0, 0, 0); db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid); manifest_crosslink(nrid,&wiki); blob_reset(&wiki); content_deltify(rid,nrid,0); db_end_transaction(0); Index: src/xfer.c ================================================================== --- src/xfer.c +++ src/xfer.c @@ -27,11 +27,11 @@ typedef struct Xfer Xfer; struct Xfer { Blob *pIn; /* Input text from the other side */ Blob *pOut; /* Compose our reply here */ Blob line; /* The current line of input */ - Blob aToken[5]; /* Tokenized version of line */ + Blob aToken[6]; /* Tokenized version of line */ Blob err; /* Error message text */ int nToken; /* Number of tokens in line */ int nIGotSent; /* Number of "igot" cards sent */ int nGimmeSent; /* Number of gimme cards sent */ int nFileSent; /* Number of files sent */ @@ -132,20 +132,20 @@ pXfer->nDeltaRcvd++; }else{ srcid = 0; pXfer->nFileRcvd++; } - rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid); + rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid, 0); remote_has(rid); blob_reset(&content); return; } if( pXfer->nToken==4 ){ Blob src, next; srcid = rid_from_uuid(&pXfer->aToken[2], 1); if( content_get(srcid, &src)==0 ){ - rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid); + rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid, 0); pXfer->nDanglingFile++; db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid); content_make_public(rid); return; } @@ -159,20 +159,79 @@ } sha1sum_blob(&content, &hash); if( !blob_eq_str(&pXfer->aToken[1], blob_str(&hash), -1) ){ blob_appendf(&pXfer->err, "content does not match sha1 hash"); } - rid = content_put(&content, blob_str(&hash), 0); + rid = content_put(&content, blob_str(&hash), 0, 0); blob_reset(&hash); if( rid==0 ){ blob_appendf(&pXfer->err, "%s", g.zErrMsg); }else{ content_make_public(rid); manifest_crosslink(rid, &content); } remote_has(rid); } + +/* +** The aToken[0..nToken-1] blob array is a parse of a "cfile" line +** message. This routine finishes parsing that message and does +** a record insert of the file. The difference between "file" and +** "cfile" is that with "cfile" the content is already compressed. +** +** The file line is in one of the following two forms: +** +** cfile UUID USIZE CSIZE \n CONTENT +** cfile UUID DELTASRC USIZE CSIZE \n CONTENT +** +** The content is CSIZE bytes immediately following the newline. +** If DELTASRC exists, then the CONTENT is a delta against the +** content of DELTASRC. +** +** The original size of the UUID artifact is USIZE. +** +** If any error occurs, write a message into pErr which has already +** be initialized to an empty string. +** +** Any artifact successfully received by this routine is considered to +** be public and is therefore removed from the "private" table. +*/ +static void xfer_accept_compressed_file(Xfer *pXfer){ + int szC; /* CSIZE */ + int szU; /* USIZE */ + int rid; + int srcid = 0; + Blob content; + + if( pXfer->nToken<4 + || pXfer->nToken>5 + || !blob_is_uuid(&pXfer->aToken[1]) + || !blob_is_int(&pXfer->aToken[pXfer->nToken-2], &szU) + || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &szC) + || szC<0 || szU<0 + || (pXfer->nToken==5 && !blob_is_uuid(&pXfer->aToken[2])) + ){ + blob_appendf(&pXfer->err, "malformed cfile line"); + return; + } + blob_zero(&content); + blob_extract(pXfer->pIn, szC, &content); + if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){ + /* Ignore files that have been shunned */ + return; + } + if( pXfer->nToken==5 ){ + srcid = rid_from_uuid(&pXfer->aToken[2], 1); + pXfer->nDeltaRcvd++; + }else{ + srcid = 0; + pXfer->nFileRcvd++; + } + rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid, szC); + remote_has(rid); + blob_reset(&content); +} /* ** Try to send a file as a delta against its parent. ** If successful, return the number of bytes in the delta. ** If we cannot generate an appropriate delta, then send @@ -333,10 +392,55 @@ } } remote_has(rid); blob_reset(&uuid); } + +/* +** Send the file identified by rid as a compressed artifact. Basically, +** send the content exactly as it appears in the BLOB table using +** a "cfile" card. +*/ +static void send_compressed_file(Xfer *pXfer, int rid){ + const char *zContent; + const char *zUuid; + char *zDelta; + int szU; + int szC; + int rc; + Stmt s; + + db_prepare(&s, + "SELECT uuid, size, content FROM blob" + " WHERE rid=%d" + " AND size>=0" + " AND uuid NOT IN shun" + " AND rid NOT IN private", + rid + ); + rc = db_step(&s); + if( rc==SQLITE_ROW ){ + zUuid = db_column_text(&s, 0); + szU = db_column_int(&s, 1); + szC = db_column_bytes(&s, 2); + zContent = db_column_raw(&s, 2); + zDelta = db_text(0, "SELECT uuid FROM blob WHERE rid=" + " (SELECT srcid FROM delta WHERE rid=%d)", rid); + blob_appendf(pXfer->pOut, "cfile %s ", zUuid); + if( zDelta ){ + blob_appendf(pXfer->pOut, "%s ", zDelta); + fossil_free(zDelta); + pXfer->nDeltaSent++; + }else{ + pXfer->nFileSent++; + } + blob_appendf(pXfer->pOut, "%d %d\n", szU, szC); + blob_append(pXfer->pOut, zContent, szC); + blob_append(pXfer->pOut, "\n", 1); + } + db_finalize(&s); +} /* ** Send a gimme message for every phantom. ** ** It should not be possible to have a private phantom. But just to be @@ -511,11 +615,11 @@ nRow++; if( nRow>=800 && nUncl>nRow+100 ){ md5sum_blob(&cluster, &cksum); blob_appendf(&cluster, "Z %b\n", &cksum); blob_reset(&cksum); - content_put(&cluster, 0, 0); + content_put(&cluster, 0, 0, 0); blob_reset(&cluster); nUncl -= nRow; nRow = 0; } } @@ -523,11 +627,11 @@ db_multi_exec("DELETE FROM unclustered"); if( nRow>0 ){ md5sum_blob(&cluster, &cksum); blob_appendf(&cluster, "Z %b\n", &cksum); blob_reset(&cksum); - content_put(&cluster, 0, 0); + content_put(&cluster, 0, 0, 0); blob_reset(&cluster); } } } @@ -666,10 +770,31 @@ @ error %T(blob_str(&xfer.err)) nErr++; break; } }else + + /* cfile UUID USIZE CSIZE \n CONTENT + ** cfile UUID DELTASRC USIZE CSIZE \n CONTENT + ** + ** Accept a file from the client. + */ + if( blob_eq(&xfer.aToken[0], "cfile") ){ + if( !isPush ){ + cgi_reset_content(); + @ error not\sauthorized\sto\swrite + nErr++; + break; + } + xfer_accept_compressed_file(&xfer); + if( blob_size(&xfer.err) ){ + cgi_reset_content(); + @ error %T(blob_str(&xfer.err)) + nErr++; + break; + } + }else /* gimme UUID ** ** Client is requesting a file. Send it. */ @@ -763,14 +888,21 @@ if( xfer.nToken==3 && blob_is_int(&xfer.aToken[1], &iVers) && iVers>=2 ){ int seqno, max; + if( iVers>=3 ){ + cgi_set_content_type("application/x-fossil-uncompressed"); + } blob_is_int(&xfer.aToken[2], &seqno); max = db_int(0, "SELECT max(rid) FROM blob"); while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){ - send_file(&xfer, seqno, 0, 1); + if( iVers>=3 ){ + send_compressed_file(&xfer, seqno); + }else{ + send_file(&xfer, seqno, 0, 1); + } seqno++; } if( seqno>=max ) seqno = 0; @ clone_seqno %d(seqno) }else{ @@ -1032,11 +1164,11 @@ /* ** Always begin with a clone, pull, or push message */ if( cloneFlag ){ - blob_appendf(&send, "clone 2 %d\n", cloneSeqno); + blob_appendf(&send, "clone 3 %d\n", cloneSeqno); pushFlag = 0; pullFlag = 0; nCardSent++; /* TBD: Request all transferable configuration values */ content_enable_dephantomize(0); @@ -1191,10 +1323,19 @@ ** Receive a file transmitted from the server. */ if( blob_eq(&xfer.aToken[0],"file") ){ xfer_accept_file(&xfer, cloneFlag); }else + + /* cfile UUID USIZE CSIZE \n CONTENT + ** cfile UUID DELTASRC USIZE CSIZE \n CONTENT + ** + ** Receive a compressed file transmitted from the server. + */ + if( blob_eq(&xfer.aToken[0],"cfile") ){ + xfer_accept_compressed_file(&xfer); + }else /* gimme UUID ** ** Server is requesting a file. If the file is a manifest, assume ** that the server will also want to know all of the content files @@ -1249,11 +1390,11 @@ } if( zPCode==0 ){ zPCode = mprintf("%b", &xfer.aToken[2]); db_set("project-code", zPCode, 0); } - blob_appendf(&send, "clone 2 %d\n", cloneSeqno); + blob_appendf(&send, "clone 3 %d\n", cloneSeqno); nCardSent++; }else /* config NAME SIZE \n CONTENT ** @@ -1363,20 +1504,20 @@ break; } }else /* Unknown message */ - { + if( xfer.nToken>0 ){ if( blob_str(&xfer.aToken[0])[0]=='<' ){ fossil_warning( "server replies with HTML instead of fossil sync protocol:\n%b", &recv ); nErr++; break; } - blob_appendf(&xfer.err, "unknown command: %b", &xfer.aToken[0]); + blob_appendf(&xfer.err, "unknown command: [%b]", &xfer.aToken[0]); } if( blob_size(&xfer.err) ){ fossil_warning("%b", &xfer.err); nErr++;