From c897c515e749388d138db741ca1701fe6be15792 Mon Sep 17 00:00:00 2001 From: Moritz Christian Weber Date: Thu, 8 Sep 2022 02:01:27 +0200 Subject: [PATCH] Fully commented version of very small git clone. It has 1 or more bugs (most likely memory-related) in the delta-resolving. I am sure that it is memory-related and in the resolving , because I concatenate deltas and the result is slightly broken, which creates wrong hashes and confused the resolving further. Repos & versions without deltas work beautifully (i.e. toybox 0.0.1 which is the only version of toybox without deltas). Due to 3 kinds of indirections (hashes-references, deltas and recursive resolving) it is hard to test and to debug. In addition there are not reference examples to write generic test cases for delta resolving. --- toys/pending/git.c | 726 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 574 insertions(+), 152 deletions(-) diff --git a/toys/pending/git.c b/toys/pending/git.c index 969cd1b7..d2993808 100644 --- a/toys/pending/git.c +++ b/toys/pending/git.c @@ -9,8 +9,17 @@ * https://git-scm.com/docs/pack-format * https://git-scm.com/docs/index-format * https://www.alibabacloud.com/blog/a-detailed-explanation-of-the-underlying-data-structures-and-principles-of-git_597391 - -USE_GITCLONE(NEWTOY(gitclone, ">1", TOYFLAG_USR|TOYFLAG_BIN)) + * https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt + * https://stackoverflow.com/a/14303988 + * https://stackoverflow.com/a/21599232 + * https://github.com/tarruda/node-git-core/blob/master/src/js/delta.js + + +USE_GITCLONE(NEWTOY(gitclone, "<1", TOYFLAG_USR|TOYFLAG_BIN)) +USE_GITINIT(NEWTOY(gitinit, "<1", TOYFLAG_USR|TOYFLAG_BIN)) +USE_GITREMOTE(NEWTOY(gitremote, "<1", TOYFLAG_USR|TOYFLAG_BIN)) +USE_GITFETCH(NEWTOY(gitfetch, 0, TOYFLAG_USR|TOYFLAG_BIN)) +USE_GITCHECKOUT(NEWTOY(gitcheckout, "<1", TOYFLAG_USR|TOYFLAG_BIN)) config GITCOMPAT bool "gitcompat" @@ -24,28 +33,28 @@ config GITCLONE help usage: gitclone URL A minimal git clone. - + config GITINIT bool "gitinit" default n help usage: gitinit NAME A minimal git init. - + config GITREMOTE bool "gitremote" default n help usage: gitremote URL A minimal git remote add origin. - + config GITFETCH bool "gitfetch" default n help usage: gitfetch A minimal git fetch. - + config GITCHECKOUT bool "gitcheckout" default n @@ -54,207 +63,620 @@ config GITCHECKOUT A minimal git checkout. */ -#define FOR_git -#include TT git +#define TT this.git +#define FOR_gitclone #include "toys.h" +#include "openssl/sha.h" //ToDo: borrowed from OpenSSL to not pipe or refactor the SHA1SUM in toybox +#include "zlib.h" //ToDo: borrowed from libz to not refactor deflate.c GLOBALS( char *url; - char *name; + char *name; + struct IndexV2 *i; ) -struct IndexV2 { - char header[8] - long fot[256] - char *sha1[20] - long *crc - long *offset - long long *64_offset - char packsha1[20] - char idxsha1[20] +struct IndexV2 { //git inxed format v2 https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt#L241 + char header[8]; + uint32_t fot[256]; + char (*sha1)[20]; + uint32_t *crc; + uint32_t *offset; + long long *offset64;//not supported yet + char packsha1[20]; + char idxsha1[20]; }; - -static void read_index(FILE *fpi, struct IndexV2 *i){ - fpi=fopen(".git/objects/pack/temp.idx","wb"); - fread(i->header, sizeof(index->header),1,fpi) - fread(i->fot, sizeof(index->fot),1,fpi); - fread(i->sha1, sizeof(index->fot),fot[255],fpi); - fread(i->crc, sizeof(index->fot),fot[255],fpi); - fread(i->offset, sizeof(index->fot),fot[255],fpi); - //TODO: Offsets for file size 2G missing here - if fread(i->packsha1, 20,1,fpi); - fread(i->idxsha1, 20,1,fpi); -} -void write_children(char *hash, char* path, FILE *fpp){ - FILE* fc; - - fseek(fpp,get_index(Index* i,hash),SEEK_SET); - fread(object, unpack(fpp, type, offset),1,fpp); - //inflate(object,bb) //TODO: Inflate & Filter - if (strncmp(object[],"40",4)){//tree - for each object.subfolders{ - write_children(child->hash,strcat(strcat(path,"/"),),fpp); - } - for each object.files{ - write_children(child->hash,strcat(strcat(path,"/"),),fpp; - } - } else if (strncmp(object,"100",4)){//file - fc = fopen(path, "w"); - fwrite(object,1,sizeof(object),fc); - fclose(fc); - return; +static void read_index(struct IndexV2 *i){ + FILE *fpi; + i=malloc(sizeof(i)); + //i->fot={ 0 }; + i->sha1=malloc(20*sizeof(char)); + i->crc=malloc(sizeof(uint32_t)); + i->offset=malloc(sizeof(uint32_t)); + i->offset64=malloc(sizeof(long long)); + if (access(".git/object/pack/temp.idx", F_OK)==0) {//TODO: not used yet as index is not persisted yet + fpi=fopen(".git/object/pack/temp.idx","rb");//persistance needed for other git commands (not clone) + printf("read header\n"); + fread(i->header, sizeof(i->header),1,fpi); + printf("Header: %s..Read fot\n",i->header); + fread(i->fot, 4,256,fpi); + printf("Elements %d..Read sha1\n",i->fot[255]); + fread(i->sha1, sizeof(i->fot),i->fot[255],fpi); + printf("read crc\n"); + fread(i->crc, sizeof(i->fot),i->fot[255],fpi); + printf("read offset\n"); + fread(i->offset, sizeof(i->fot),i->fot[255],fpi); + //TODO: Offsets for file size 2G missing here + printf("read packsha\n"); + fread(i->packsha1, 20,1,fpi); + printf("read idxsha\n"); + fread(i->idxsha1, 20,1,fpi); + fclose(fpi); } } -uint64_t unpack( const uint8_t *const fp, char *type, long *offset) -{ - int i,bitshift,data= 0; +static char *l;//for saving the insertation position + +int cmp (const void *i, void *j){ + l=j; //inject inseration position in compare to binary search + //printf("Compare %p %p %d\n",i,j,strncmp(i,j,20)); + return strncmp(i,j,20); +} + +long get_index(struct IndexV2 *i, char *h){ + char *pos=(char *)bsearch(h, i->sha1[0], i->fot[255], 20, + (int(*)(const void*,const void*)) cmp);//TODO: Should be placed by bsearchpos() below; cmp and *l to be removed too + for (int j=0;j<20;j++) printf("%02x",h[j]); + printf("\n"); + printf("index pointer: %p\n",pos); + printf("fot[255]: %d\n",i->fot[255]); + printf("sha1[0] pointer: %p\n",i->sha1[0]); + printf("offset index : %ld\n",(pos-i->sha1[0])/20); + return i->offset[(pos-i->sha1[0])/20]; +} + +//read type and lenght of an packed object +uint64_t unpack(FILE *fpp, int *type, long int *offset) +{//https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt#L30 +//https://yqintl.alicdn.com/eef7fe4f22cc97912cee011c99d3fe5821ae9e88.png + int bitshift= 4; uint64_t length = 0; - - fread(data, 1,1,fpi); - type=data & 0x70; - while((data & 0x80) != 0 && fread(data, 1,1,fpi))!=1) + uint8_t data; + printf("Start unpack\n"); + fseek(fpp,*offset,SEEK_SET); + printf("Offset set to: %ld\n",*offset); + fread(&data, 1,1,fpp); + printf("Data: %d\n",data); + *type=((data & 0x70)>>4); + printf("Type: %d\n",*type); + length|= (uint64_t)(data & 0x0F); + //(*offset)++; + while((data & 0x80) && fread(&data, 1,1,fpp)!=-1) { - length |= (uint64_t)(data & 0x7F) << bitshift; - bitshift += 7; i++; - } - offset = i; + length |= (uint64_t)(data & 0x7F) << bitshift; + bitshift += 7;// (*offset)++; + //printf("Offset set to: %ld\n",*offset); + } + //printf("Offset set to: %ld\n",*offset); + printf("Length: %ld\n",length); return length; } -void get_index(Index* i, char* []h){ - return i->offset[bsearch(h, i->sha, i->fot[h[0]]-i->fot[h[0]]-1, 20, - (int(*)(const void*,const void*)) strcmp)]; +// ToDo: borrowed from int inf(FILE *source, FILE *dest) in +// zpipe.c: example of proper use of zlib's inflate() and deflate() +// Not copyrighted -- provided to the public domain +// Version 1.4 11 December 2005 Mark Adler */ +#define CHUNK 4096 +int inf(FILE *source, char *dest) //modified signature to ease use +{ + int ret; + char *position=dest; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); + if (ret != Z_OK) + return ret; + + // decompress until deflate stream ends or end of file + do { + + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + if (strm.avail_in == 0) + break; + strm.next_in = in; + + + // run inflate() on input until output buffer not full + do { + + strm.avail_out = CHUNK; + strm.next_out = out; + + ret = inflate(&strm, Z_NO_FLUSH); + //assert(ret != Z_STREAM_ERROR); // state not clobbered + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; // and fall through + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + + have = CHUNK - strm.avail_out; + memcpy(position,out,have);//added to original + position+=have;//added to original + //if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + // (void)inflateEnd(&strm); + // return Z_ERRNO; + //} + } while (strm.avail_out == 0); + // done when inflate() says it's done + } while (ret != Z_STREAM_END); + fseek(source,ftell(source)-strm.avail_in,SEEK_SET);//modified from zpipe.c to set FP to end of zlib object + // clean up and return + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + +//inspired by musl bsearch +long bsearchpos(const void *k, const void *a,size_t h, size_t w){ +long l=0,m=0,r=0; +if (!h) return 0; +//printf("Array: %p Key:%p\n",a,k); +while(h>0){ + m=l+(h/2); + //m=(l+h)/2; + //printf("l: %ld m:%ld h:%ld\n",l,m,h); + r=strncmp(k,a+(m*w),20); + //printf("r: %ld\n",r); + if(!r||h==1)break;//match on search or position for insert + if(r<0){h/=2;}else{l=m;h-=h/2;} + //if(r<0){h=m-1;}else{l=m+1;} +} + //printf("Return m: %ld r:%ld \n",m,r); + +return m+=(r>0)?1:0;//For inserts check if insert is bigger obj at identified position } -void set_object(struct Index* idx, char* o, long offset){ - char *h; - //TODO: hash=sha1(o); - idx->offset[idx->fot[h[0]]]=offset; +long set_object(struct IndexV2 *idx,int type, char *o, uint32_t count, uint32_t ofs){ + printf("Alloc... ");//TODO: Too many allocs in here 1) to concat the search string for hashing 2) to insert into the array (can be reduce to a single malloc in fetch as the pack header contains the number of objects in pack + char *c,*p="",*h=(char*)xmalloc(sizeof(char)*20);//composition,prefix,hash + long pos=0; + switch(type) + {//https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt#L49 + case 1:p=xmprintf("commit %d",count);break;//count is used as o can contain \0 in the string + case 2:p=xmprintf("tree %d",count);break; + case 3:p=xmprintf("blob %d",count);break; + case 4:p=xmprintf("tag %d",count);break; + case 6:printf("REF_DELTA");break;//not expected in fetch packs as fetch packs are self-containing + case 7:printf("OBJ_DELTA\n");break; + } + c=(char*)xmalloc(strlen(p)+count+2);//Robs null terminator embedding + memcpy(c,p,strlen(p)+1);//Robs null terminator embedding + memcpy(c+strlen(p)+1,o,count+1);//Robs null terminator embedding + //printf("Enriched Object: %s %ld\n",c,strlen(p)+count+2); + h=SHA1(c,strlen(p)+count+1,h);//ToDo: borrowed from OpenSSL to not to pipe or refactor SHA1SUM in toybox + printf("..Binary search\n"); + //printf("\nidx->fot[255]=%d\n",idx->fot[255]); + //printf("idx->sha1[fot[h[0]]]=%d\n",sizeof(idx->sha1)); + //TODO:Array Insert broken + //if (idx->fot[255]>1) + //{ + // printf("Bsearch result: %p\n",bsearch(h, idx->sha1[0], idx->fot[255], 20, + // (int(*)(const void * ,const void *)) cmp));//find insertation position + // printf("Inseration position pointer %p\n",l); + // pos=(long)(((l-idx->sha1[0])/20)+((strncmp(h,l,20)<0)?0:1));//ugly ins pos hack + // printf("Bigger one %ld\n",(long)(((l-idx->sha1[0])/20)+((strncmp(h,l,20)<0)?0:1))); + //} else { + // printf("Smaller two\n"); + // if (idx->fot[255]==0) + // { + // pos=0; + // }else{ + // pos=(strncmp(h,idx->sha1[0],20)<0)?0:1; + // } + // } + //printf("Binary search position: %ld, %p %p\n",pos,idx->sha1[0],l); + //l=NULL; + for (int j=0;j<20;j++) printf("%02x",h[j]);//find insert position + pos=bsearchpos(h,idx->sha1[0],idx->fot[255],20); + printf("\n..Insert pos %ld\n",pos); + printf("..Preloop\n"); + for (int i=h[0];i<=255;i++) + idx->fot[i]+=1; //adjust of fanout table https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt#L179 + printf("Post loop\n"); + printf("Resize sha1 array..idx->fot[255]%d\n",idx->fot[255]); //Memory management for insert TODO:Could be also a single malloc at gitfetch based on the nbr of objects in pack + idx->sha1=realloc(idx->sha1,(idx->fot[255]+1)*20*sizeof(char));//Did not fix the TODO yet, because set_object could be reused for other command im mem mgmt is here + printf("Mem copy sha1 array..sizeof(idx->sha1)%ld\n",sizeof(idx->sha1)); + memmove(&idx->sha1[pos+1],&idx->sha1[pos],(idx->fot[255]-pos)*20*sizeof(char)); + printf("Resize offset\n"); + idx->offset=realloc(idx->offset,(idx->fot[255]+1)*sizeof(uint32_t)); + printf("Mem copy offset\n"); + memmove(&idx->offset[pos+1],&idx->offset[pos],sizeof(uint32_t)*(idx->fot[255]-pos)); + printf("Set offset value\n"); + memcpy(&idx->sha1[pos],h,20);//insert SHA1 + idx->offset[pos]=ofs;//insert offset of SHA1 //ToDo: id->crc[idx->fot[h[0]]]=; - for (int i=hash[0];i<255;i++)idx->fot[i]]+=1; + printf("Write object\n"); + // printf("SetGet %d %ld\n :",idx->offset[pos],get_index(idx,h)); //writeObject to local pack; + // for (int h=0;hfot[255];h++){ + // for (int j=0;j<20;j++) printf("%02x",idx->sha1[h][j]); + // printf("\n"); + // } + free(h); + free(c); + return ofs; } -static void git_init(char *name) +static void gitinit(char *name) { - mkdir(name,0755); - mkdir(xmprintf("%s%s",name,"/.git"),0755); - mkdir(xmprintf("%s%s",name,".git/objects"),0755); - mkdir(xmprintf("%s%s",name,".git/objects/pack"),0755); - mkdir(xmprintf("%s%s",name,"/.git/branches"),0755); - mkdir(xmprintf("%s%s",name,"/.git/hooks"),0755);//hook files skiped - mkdir(xmprintf("%s%s",name,"/.git/info"),0755); - mkdir(xmprintf("%s%s",name,".git/objects/info"),0755); - mkdir(xmprintf("%s%s",name,".git/refs"),0755); - mkdir(xmprintf("%s%s",name,".git/heads"),0755); - mkdir(xmprintf("%s%s",name,".git/tags"),0755); - xcreate(xmprintf("%s%s",name,".git/config"),O_CREAT,0644); - xcreate(xmprintf("%s%s",name,".git/description"),O_CREAT,0644); - xcreate(xmprintf("%s%s",name,".git/HEAD"),O_CREAT,0644); - xcreate(xmprintf("%s%s",name,".git/info/exclude"),O_CREAT,0644); + if (mkdir(name,0755)!=0){//For git clone actually only refs and object/pack needed + mkdir(xmprintf("%s%s",name,"/.git"),0755);//I create the other for a git compliant folder structure + mkdir(xmprintf("%s%s",name,"/.git/objects"),0755); + mkdir(xmprintf("%s%s",name,"/.git/objects/pack"),0755); + mkdir(xmprintf("%s%s",name,"/.git/branches"),0755); + mkdir(xmprintf("%s%s",name,"/.git/hooks"),0755);//hook files skipped as implementations does not support hooks + mkdir(xmprintf("%s%s",name,"/.git/info"),0755); + mkdir(xmprintf("%s%s",name,"/.git/objects/info"),0755); + mkdir(xmprintf("%s%s",name,"/.git/refs"),0755); + mkdir(xmprintf("%s%s",name,"/.git/heads"),0755); + mkdir(xmprintf("%s%s",name,"/.git/tags"),0755); + xcreate(xmprintf("%s%s",name,"/.git/config"),O_CREAT,0644); + xcreate(xmprintf("%s%s",name,"/.git/description"),O_CREAT,0644); + xcreate(xmprintf("%s%s",name,"/.git/HEAD"),O_CREAT,0644); + xcreate(xmprintf("%s%s",name,"/.git/info/exclude"),O_CREAT,0644); + } } -static void git_remote_add_origin(char *url) +static void gitremote(char *url) { - FILE *fp; - fp=fopen(".git/config","wb"); - fwrite("[core]\n",1,7,fp); - fwrite("\trepositoryformatversion = 0\n",1,29,fp); - fwrite("\tfilemode = false\n",1,18,fp); - fwrite("\tbare = false\n",1,14,fp); - fwrite("\tlogallrefupdates = true\n",1,25,fp); - fwrite("\tsymlinks = false\n",1,18,fp); - fwrite("\tignorecase = true\n",1,19,fp); - fwrite("[remote \"origin\"]\n",1,18,fp); - fwrite(xmprintf("%s%s","\turl = %s/refs\n",TT->url),1,sizeof(TT->url)+13,fp); - fwrite("\tfetch = +ref/heads/*:refs/remotes/origin/*\n",1,44,fp); - fclose(fp); + if (access(".git/config", F_OK)!=0) { + FILE *fp; + fp=fopen(".git/config","wb"); + fwrite("[core]\n",1,7,fp); + fwrite("\trepositoryformatversion = 0\n",1,29,fp); + fwrite("\tfilemode = false\n",1,18,fp); + fwrite("\tbare = false\n",1,14,fp); + fwrite("\tlogallrefupdates = true\n",1,25,fp); + fwrite("\tsymlinks = false\n",1,18,fp); + fwrite("\tignorecase = true\n",1,19,fp); + fwrite("[remote \"origin\"]\n",1,18,fp); + fwrite(xmprintf("\turl = %s/refs\n",TT.url),1,strlen(TT.url)+13,fp); + fwrite("\tfetch = +ref/heads/*:refs/remotes/origin/*\n",1,44,fp); + fclose(fp); + } } -static void git_fetch(void) +//this is most likely still buggy and create a late observable heap overflow larger deltafied repos +char* resolve_delta(char *s, char *d, long dsize, uint32_t *count) +{ //https://stackoverflow.com/a/14303988 + long pos=0,bitshift=0; + printf("Original Source: \n"); + //for (int k=0;k<*count;k++){printf("%c",s[k]);} + //printf("\n"); + //printf("Delta:\n"); + //for (int k=0;kfot[255];h++){ + // for (int j=0;j<20;j++) printf("%02x",i->sha1[h][j]); + // printf("\n"); + //} + free(h); + return resolve_delta(source,object,dcount,count);//recursion due to https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt#L83 + } else { + printf("Type Else:\n"); + inf(fpp,object); + *count=dcount; + printf("Type Else end:\n"); + return object; + } +} + +char* txtoh(char *p){ //make 20byte SHA1 hash from 40 byte SHA1 string +// printf("txtoh start"); + char *h=malloc(sizeof(char)*41); //TODO: Dont like the malloc here, but did not find a solution to sscanf into p again + for(int c=0; c<20;c++){ + // printf("c: %d\n",c); + sscanf(&p[2*c],"%2hhx",&(h[c])); + } +// printf("txtoh end"); + h[20]='\0'; + printf("return"); + return h; +} + +//traveres the commit tree for checkout +void write_children(char *hash, char *path, FILE *fpp){ + FILE *fc; + char *object; + int type; + long int offset; + uint32_t count; + printf("seek index\n"); + + offset= get_index(TT.i,hash); + printf("Found index: %ld\n",offset); + //fseek(fpp,offset,SEEK_SET); + printf("read object\n"); + // size_t size=unpack(fpp,&type,&offset); + // printf("Size: %ld \n",size); + // printf("Size2: %ld \n",size); + object=unpack_object(fpp,TT.i,offset,&count,&type); + printf("%s\n",object); + printf("Type %d\n",type); + if(type==1){//at commit object + memcpy(hash,&object[5],40); + write_children(txtoh(hash),path,fpp); + }else if(type==2){//at tree object https://stackoverflow.com/a/21599232 + char *hs=0; + int pos=0; + printf("process folder %s\n",path); + while (pos0)?xmprintf("%s/%s",path,object+pos+7):object+pos+7;//concat file name + }else{//tree object reference is a folder + name=(strlen(path)>0)?xmprintf("%s/%s",path,object+pos+6):object+pos+6;//concat folder name + printf("create folder %s\n",name); + mkdir(name,0755);//TODO: umask + } + memcpy(hash,hs,20); + write_children(hash,name,fpp); + //free(name); + pos=hs-object+20; + } + }else{//at blob/file object + printf("process file %s\n",path); + fc = fopen(path, "w"); + printf("process opened \n"); + fputs(object,fc);//TODO:Not sure if length might be an issue here + printf("process file written\n"); + fclose(fc); + } +free(object); +printf("Child: %s done\n",path); +} + +static void gitfetch(void) { - execl("toybox",{"-O" ".git/refs/temp.refs" "https://github.com/landley/toybox/info/refs?service=git-upload-pack"});//TODO: Refactor wget into lib - execl("toybox",{"wget","-O",".git/objects/pack/temp.pack","-d","-p","$'0032want 8cf1722f0fde510ea81d13b31bde1e48917a0306\n00000009done\n' https://github.com/landley/toybox/git-upload-pack}");//TODO: does not skip 0008NAK - FILE *fpp,*fpi; - read_index(fpi,i); + //size_t l=0; + printf("refs\n"); + pid_t pid; //TODO:I use herein after two temp files for fetch which git due not offer to 1) avoid a rewrite and 2) messing up the repo files while testing + if ((pid=fork())==0)execv("toybox",(char *[]){"toybox","wget","-O",".git/refs/temp.refs","https://github.com/landley/toybox/info/refs?service=git-upload-pack",(char*)0});//TODO: Refactor wget into lib + perror("execv\n"); + // char h[]="8cf1722f0fde510ea81d13b31bde1e48917a0306"; + char h[]="52fb04274b3491fdfe91b2e5acc23dc3f3064a86";//TODO: Replace static testing hash and uncomment the following line if rare delta resolve /?heap overflow? bug was found + //FILE *fpr; + //fpr=fopen(".git/ref/temp.refs","r"); + //fseek(); + //getline(&h,&l,fpr); + //getline(&h,&l,fpr); + //getline(&h,&l,fpr); + //fclose(fpr); + //strcpy(h,&h[4],4); + //h[40]='\0'; + printf("pack\n"); + //if ((pid=fork())==0)execv("toybox",(char *[]){"toybox","wget","-O",".git/objects/pack/temp.pack","-p","$'0032want 52fb04274b3491fdfe91b2e5acc23dc3f3064a86\n00000009done\n'","https://github.com/landley/toybox/git-upload-pack",(char*)0});//TODO: does not skip 0008NAK printf("init\n"); + if ((pid=fork())==0)execv("toybox",(char *[]){"toybox","wget","-O",".git/objects/pack/temp.pack","-p",xmprintf("$'0032want %s\n00000009done\n'",h),"https://github.com/landley/toybox/git-upload-pack",(char*)0});//TODO: does not skip 0008NAK printf("init\n"); + perror("execv\n"); + FILE *fpp; + printf("openpack\n"); fpp=fopen(".git/objects/pack/temp.pack","r"); - long *offset; - long *ocount; - char *type; - fsetpos(fpp,16)//header check skipped - fread(ocount,4,1,fpp) - while (feof(fpp)){ - fread(&object, unpack(fpp, type, offset),1,fpp); - set_object(object,ftell(fpp)) + printf("read index\n"); + read_index(TT.i);//init index with out reading + printf("init\n"); + uint32_t ocount=0, count; + int type; + char *object; + printf("skip header\n"); + long int offset=12+8;//8byte from the wget post response are skipped too + fseek(fpp,8+8,SEEK_SET);//header check skipped https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt#L14 + printf("read count\n"); + fread(&ocount,4,1,fpp); + ocount=ntohl(ocount);//https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt#L21 + printf("Count: %d ..Loop pack\n",ocount); + for (int j=0;j5126)printf("SetGetObject %d: %s\n",j,object); + //printf("SetGetUnpack %d: %s\n",j,unpack_object(fpp,TT.i,set_object(TT.i,type,object,count,offset),&count,&type)); + set_object(TT.i,type,object,count,offset); + + free(object); + + printf("Adjust offset\n"); + offset=ftell(fpp);//adjust offset to new file position + printf("Adjusted offset to: %ld\n",offset); } - fclose(fpi); - flcode(fpp); + //for (int h=0;hfot[255];h++){ + //printf("%d: ",h); + //for (int j=0;j<20;j++) printf("%02x",TT.i->sha1[h][j]); + // printf("\n"); + // } + + //TODO: Final pack checksum not calculated and checked + fclose(fpp); } -static void git_checkout(char *name) +static void gitcheckout(char *name) { - FILE *fpp,*fh; - char *hash - - fh=fopen(strcat("./ref/heads/",!hash?"master":name),"r"); - fread(hash,20,1,fh); - fclose(fh); + + FILE *fpp; + //FILE *fh; + printf("Find branch for checkout\n"); + //fh=fopen(xmprintf(".git/ref/heads/%s",name?"master":name),"r");//TODO: Checkout master as in ref/heads + printf("Read head\n"); + //hf[fread(&hf,40,1,fh)]='\0'; + //fclose(fh); + printf("Close heads and read pack\n"); fpp=fopen(".git/objects/pack/temp.pack","r"); - write_children(hash,"",ffp); - fclose(ffp); + printf("set signature\n"); + char *p="52fb04274b3491fdfe91b2e5acc23dc3f3064a86";//static hashes for testing 3604ba4f42c3d83e2b14f6d0f423a33a3a8706c3"; + //char *p="8cf1722f0fde510ea81d13b31bde1e48917a0306";//3604ba4f42c3d83e2b14f6d0f423a33a3a8706c3"; + printf("enter tree root\n"); + write_children(txtoh(p),"",fpp); + fclose(fpp); } -void git_clone_main(void) +void gitclone_main(void) { - TT.url = xstrdup(toys.optargs[0])); + TT.url = xstrdup(toys.optargs[0]); if(strend(TT.url,".git")) TT.url[strlen(TT.url)-4]='\0'; - TT.name = strrchr(TT.url,'/')+1: - git_init(TT.name); + TT.name = strrchr(TT.url,'/')+1; + gitinit(TT.name); chdir(TT.name); - git_remote(TT.url); - git_fetch(); - git_checkout("master"); + TT.i=malloc(sizeof(struct IndexV2)); + gitremote(TT.url); + gitfetch(); + gitcheckout("master"); chdir(".."); + return; } -void git_init_main(void) +#define FOR_gitinit +#include "generated/flags.h" + +void gitinit_main(void) { - git_init(strrchr(strtok(xstrdup(toys.optargs[0]),"."),'/');); + gitinit(xstrdup(toys.optargs[0])); } -void git_remote_main(void) +#define FOR_gitremote + +void gitremote_main(void) { - git_remote(xstrdup(toys.optargs[0])); + TT.url = xstrdup(toys.optargs[0]); + if(strend(TT.url,".git")) TT.url[strlen(TT.url)-4]='\0'; + gitremote(TT.url); } -void git_fetch_main(void) +#define FOR_gitinit + +void gitfetch_main(void) { -git_fetch(); + gitfetch(); } -void git_checkout_main(void) +#define FOR_gitcheckout + +void gitcheckout_main(void) { - TT.name = strtok(xstrdup(toys.optargs[0]),"."); - git_checkout(name); + gitcheckout(xstrdup(toys.optargs[0])); } + // ./toybox wget -O - -d https://github.com/landley/toybox/info/refs?service=git-upload-pack | less // ./toybox wget -O pack.dat -d -p $'0032want 8cf1722f0fde510ea81d13b31bde1e48917a0306\n00000009done\n' https://github.com/landley/toybox/git-upload-pack - - -USE_GITINIT(NEWTOY(gitinit, <1, TOYFLAG_USR|TOYFLAG_BIN)) -USE_GITREMOTE(NEWTOY(gitremote, <1, TOYFLAG_USR|TOYFLAG_BIN)) -USE_GITFETCH(NEWTOY(gitfetch, 0, TOYFLAG_USR|TOYFLAG_BIN)) -USE_GITCHECKOUT(NEWTOY(gitcheckout, <1, TOYFLAG_USR|TOYFLAG_BIN)) - - -long get_index(struct IndexV2 *i, char *h){ - return i->offset[bsearch(h, i->sha1, i->fot[h[0]]-i->fot[h[0]]-1, 20, (int(*)(const void*,const void*)) strcmp)]; -} -//USE_GIT_INIT(NEWTOY(git-init, 0, TOYFLAG_USR|TOYFLAG_BIN)) -//USE_GIT_REMOTE(NEWTOY(git-remote, 0, TOYFLAG_USR|TOYFLAG_BIN)) -//USE_GIT_FETCH(NEWTOY(git-fetch, 0, TOYFLAG_USR|TOYFLAG_BIN)) -//USE_GIT_CHECKOUT(NEWTOY(git-checkout, 0, TOYFLAG_USR|TOYFLAG_BIN)) -- 2.39.2