PBL libraries test version 1.0
Dec 7, 2012 1:59:21 GMT 1
Post by gcav on Dec 7, 2012 1:59:21 GMT 1
For BaCon to grow, the more software available to it the better.
In that spirit, here is a test release of the KEYFILE library. (Some functions are missing and the whole ISAM infrastructure is missing also)
It is based on Peter Graf's PBL libs.
I am no expert, so I am sure I have some misplaced or unneeded casts, variables ,pointers and what not...
Feel free to poke around the code and suggestions and improvements are welcome...
In the ZIP file attached are the lib, the PBL include file, the BaCon lib and the pbltest3 demo program. (The lib and the include file are there in case you do not want to download PBL and compile it form scratch. Be warned it has symbols and debugging items)
The test file used is the infamous worldcitiespop.txt so it will create big datafiles; more than 3,000,000 records..
The nice thing is that we can now read and write full RECORD structures to indexed files.
I will keep working on it to add the remaining functions.
gcl
In that spirit, here is a test release of the KEYFILE library. (Some functions are missing and the whole ISAM infrastructure is missing also)
It is based on Peter Graf's PBL libs.
I am no expert, so I am sure I have some misplaced or unneeded casts, variables ,pointers and what not...
Feel free to poke around the code and suggestions and improvements are welcome...
In the ZIP file attached are the lib, the PBL include file, the BaCon lib and the pbltest3 demo program. (The lib and the include file are there in case you do not want to download PBL and compile it form scratch. Be warned it has symbols and debugging items)
The test file used is the infamous worldcitiespop.txt so it will create big datafiles; more than 3,000,000 records..
The nice thing is that we can now read and write full RECORD structures to indexed files.
I will keep working on it to add the remaining functions.
gcl
-----Demo
'PBL KEYFILE Lib demo for BACON
'
' Read worldcities.txt csv file and populate a KeyFile
' You can get worldcities.tx from http://www.maxmind.com/download/worldcities/
' Do searches based on countries....
' Country,City,AccentCity,Region,Population,Latitude,Longitude
'
'
'Using 2 byte index (like country code) insert is _fast_ while
'using 40 byte Cityname index, ~3,000,000 inserts took ~25 minutes...Toshiba Tecra T2500 IntelCpu
'
'GCL
TRAP LOCAL
INCLUDE "pbl.bac"
'Datafile name
'
filename$ = "WC"
'Input CSV file
'
input$="worldcitiespop.txt"
'Some vars
'
LOCAL fp TYPE void*
LOCAL fp2 TYPE void*
LOCAL res
LOCAL point
LOCAL x,next_key_size
LOCAL temp$,searchfor$,next_key$
LOCAL rname$
'For KEY_FIND
'
LOCAL fkey$,mode$
LOCAL fkey_size,mode
'Data record for World Cities
'
RECORD wc
LOCAL name[40] TYPE char
LOCAL ct[2] TYPE char
LOCAL aname[25] TYPE char
LOCAL region
LOCAL pop
LOCAL lat TYPE float
LOCAL lon TYPE float
END RECORD
'Work in progress
'Pointer to clear out structure for each read.
'Something faster may be needed here and below
point=ADDRESS(wc)
res = K_INIT(512) :REM test buffers
'If database does not exist, create it.
IF NOT(FILEEXISTS(filename$)) THEN
fp2 = KEY_CREATE(filename$,NULL)
fp = KEY_OPEN(filename$,RW,NULL)
'Read WC csv file and insert into Key File.
OPEN input$ FOR READING AS in
WHILE NOT(ENDFILE(in)) DO
READLN data$ FROM in
SPLIT data$ BY "," TO dat$ SIZE count
'Work in progress
'Clear data record -- Take this out of here and try to do something faster
FOR x = 0 TO SIZEOF(wc)-1
POKE point + x, 0
NEXT
'Move data
K_MOVE(wc.name,dat$[1],40)
K_MOVE(wc.ct,dat$[0],2)
K_MOVE(wc.aname,dat$[2],25)
WITH wc
.region=VAL(dat$[3])
.pop=VAL(dat$[4])
.lat=VAL(dat$[5])
.lon=VAL(dat$[6])
END WITH
'Insert into KEYFILE
res = KEY_INSERT(fp,wc.name,SIZEOF(wc.name),ADDRESS(wc),SIZEOF(wc))
IF res IS 0 THEN
IF t$ <> dat$[0] THEN
t$=dat$[0]
PRINT "Inserting for Country...",dat$[0],"..."
END IF
ELSE
PRINT "Error Inserting",res
END
END IF
WEND
res = KEY_CLOSE(fp)
END IF 'FILEEXISTS
fp = KEY_OPEN(filename$,RO,NULL)
'Loop for demo reading
WHILE searchfor$ <> "exit" DO
INPUT "Search city ->",searchfor$
mode=1 :REM Default to EQ
IF searchfor$ <> "exit" THEN
INPUT "Search mode < eq | fi| la | ge | gt | le | lt>", mode$
IF mode$="eq" THEN mode=1 :REM Test find modes...
IF mode$="fi" THEN mode=2
IF mode$="la" THEN mode=3
IF mode$="ge" THEN mode=4
IF mode$="gt" THEN mode=5
IF mode$="le" THEN mode=6
IF mode$="lt" THEN mode=7
res=0
res=KEY_FIND(fp,mode,searchfor$,LEN(searchfor$),fkey$,fkey_size) :REM Add mode support
IF res < 0 THEN
PRINT "Find Error: ",res
GOTO cont
ELSE
'Clear wc record
REPEAT
FOR x = 0 TO SIZEOF(wc)-1 : REM Create function to clear RECORDS
POKE point + x, 0
NEXT
res=0
res=KEY_READ(fp,ADDRESS(wc),SIZEOF(wc))
IF res >= 0 THEN
'Print data
K_MOVE(temp$,wc.name,25)
PRINT " - ",temp$;
K_MOVE(temp$,wc.aname,25)
PRINT " - ",temp$, " ",wc.pop," ",wc.lat," ",wc.lon,"---";
K_MOVE(rname$,wc.ct,2)
PRINT "Country ",rname$
'Fetch next
K_NEXT( fp, next_key$,(void*)ADDRESS(next_key_size))
ELSE
PRINT "Read Error: ",res
END IF
UNTIL next_key$ <> searchfor$
END IF
END IF
LABEL cont
WEND
res = KEY_CLOSE(fp)
-----Library
'
' Simple PBL wrapper
' Wrapper around Peter Graf's Program Base Library
' PBL implements numerous functions, of interest here are the Key file and the ISAM procedures.
'
' May be expanded later to include other procedures
' Version 1.0: initial release
' GCL
'---------------------------------------------------------------------------------------------------------
' Requires Bacon V28 beta
' Requires PBL lib, Get from here --> http://www.mission-base.com/peter/source/
' PBL does not compile into shared lib add to makefile line below on $(THE_LIB)
' $(CC) -g -shared -Wl,-export-dynamic -Wl,-soname,libpbl.so.1 -o libpbl.so.1.0 $(LIB_OBJS)
' You also need pbl.h somewhere avaliable so that it can be passed in BaCon's command line
' All of pbl.h is avaliable with new beta.
' No longer uses IMPORT
' Required compilation with -l pbl and -i <pbl.h include>
'
'KeyFile imports
'pblKfClose close a key file
'pblKfCommit commit or rollback changes done during a transaction.
'pblKfCreate create a key file with the name specified by path.
'pblKfDelete delete the current record of the key file.
'pblKfFind find a record in a key file, set the current record
'pblKfFlush flush a key file
'pblKfGetAbs set current record to a record with an absolute position index
'pblKfGetRel set current record to a record with a relative position index
'pblKfInit change the number of cache blocks used per open key file
'pblKfInsert insert a new record with the given key and data into a key file,
'pblKfOpen open an existing key file
'pblKfRead read the data of the current record of the file
'pblKfRestorePosition restore the position of the current record saved by the last previous call to pblKfSavePositi
'KEYFILE BASE = 1000 + ---v
' PBL_ERROR_OUT_OF_MEMORY 1 -Out of memory.
' PBL_ERROR_EXISTS 2 -The record already exists.
' PBL_ERROR_NOT_FOUND 3 -Record not found.
' PBL_ERROR_BAD_FILE 4 -File structure damaged.
' PBL_ERROR_PARAM_MODE 5 -Parameter mode is not valid.
' PBL_ERROR_PARAM_KEY 6 -Parameter key is not valid.
' PBL_ERROR_PARAM_KEYLEN 7 -Parameter keylen is not valid.
' PBL_ERROR_PARAM_DATA 8 -Parameter data is not valid.
' PBL_ERROR_PARAM_DATALEN 9 -Parameter datalen is not valid.
' PBL_ERROR_PARAM_INDEX 10 -Parameter index is not valid.
' PBL_ERROR_PARAM_LIST 11 -Parameter list is not valid.
' PBL_ERROR_PARAM_COLLECTION 12 -Parameter list is not valid.
' PBL_ERROR_PARAM_ELEMENT 13 -Parameter element is not valid.
' PBL_ERROR_PARAM_SET 14 -Parameter set is not valid.
' PBL_ERROR_CREATE 20 -File system create error, see errno.
' PBL_ERROR_OPEN 21 -File system open error, see errno.
' PBL_ERROR_SEEK 22 -File system seek error, see errno.
' PBL_ERROR_READ 23 -File system read error, see errno.
' PBL_ERROR_WRITE 24 -File system write error, see errno.
' PBL_ERROR_PROGRAM 30 -An internal error in the code, debug it!!
' PBL_ERROR_NOFIT 31 -An internal condition forcing a block split.
' PBL_ERROR_NOT_ALLOWED 40 -File not open for update, operation not allowed.
' PBL_ERROR_POSITION 41 -The current record is not positioned.
' PBL_ERROR_OUT_OF_BOUNDS 42 -Index out of range (index < 0 || index >= size()).
' PBL_ERROR_CONCURRENT_MODIFICATION 43 -collection was modified concurrently to an iterator.
' FIND modes
' PBLEQ any record that is equal
' PBLFI first record that is equal
' PBLLA last record that is equal
' PBLGE last equal or first that is greater
' PBLGT first that is greater
' PBLLE first equal or last that is smaller
' PBLLT last that is smaller
'ISAM
' PBLTHIS get key and keylen of current record
' PBLNEXT get key and keylen of next record
' PBLPREV get key and keylen of previous record
' PBLFIRST get key and keylen of first record
' PBLLAST get key and keylen of last record
'pblKfFirst set the current record to the first record of the file
'pblKfLast set the current record to the last record of the file
'pblKfNext set the current record to the next record of the file
'pblKfPrev set the current record to the previous record of the file
'pblKfThis return the datalen of the current record
'Error handling:
'PBL uses response code and also pbl_errno to carry the actual error number
'This wrapper will mask the response codes, all successfull response code will be positive or 0
'If the response code is negative this will indicate the overak PBL_ERROR -(1000+X)
'Arrays start from 0
'
'
' Useful Defines/Macros/etc
'
DEF FN K_NEXT( KF, K, L ) = pblKfGetRel( (void*)KF, 1, K, L )
DEF FN K_FIRST( KF, K, L )= pblKfGetAbs( KF, 0, K, L )
DEF FN K_LAST( KF, K, L ) = pblKfGetAbs( KF, -1, K, L )
DEF FN K_PREV( KF, K, L ) = pblKfGetRel( KF, -1, K, L )
DEF FN K_THIS( KF, K, L ) = pblKfGetRel( KF, 0, K, L )
DEF FN K_INIT( N ) = pblKfInit( N )
' Kefile access mode.
CONST RO=0
CONST RW=1
'Routine to copy a string to a char array.
SUB K_MOVE(char* dest,STRING source$,NUMBER size)
LOCAL x,tmp,y
y = LEN(source$)
FOR x = 0 TO size-1
IF x > y THEN GOTO exit
tmp=ASC(MID$(source$,x+1,1))
POKE dest+x,tmp
NEXT x
LABEL exit
END SUB
'Routine to blank a structure.
'SUB K_CLEAN(NUMBER struct,NUMBER size)
' LOCAL x
' FOR x = 0 TO size-1
' POKE ADDRESS(struct)+x,0
' NEXT x
'END SUB
'
' KEY_CLOSE
'
FUNCTION KEY_CLOSE(pblKeyFile_t* fp)
LOCAL res
res = pblKfClose( fp )
IF res ISNOT 0 THEN RETURN -pbl_errno
ELSE RETURN res
END FUNCTION
'
' KEY_CREATE
'
FUNCTION KEY_CREATE(STRING name$,void* fileset)
LOCAL res TYPE void*
LOCAL res2
res = pblKfCreate( name$, fileset )
IF res IS NULL THEN
res2 = -pbl_errno
RETURN (int*)res2
ELSE
RETURN res
ENDIF
END FUNCTION
'
' KEY_OPEN
'
FUNCTION KEY_OPEN(STRING name$,NUMBER mode, void* fileset)
LOCAL res TYPE void*
LOCAL res2
res = pblKfOpen( name$, mode, fileset )
IF res IS NULL THEN
res2 = -pbl_errno
RETURN (int*)res2
ELSE
RETURN res
ENDIF
END FUNCTION
'
' KEY_INSERT
'
'res = pblKfInsert(fp ,indata.key,SIZEOF(indata.key),(void*)ADDRESS(indata),SIZEOF(indata))
FUNCTION KEY_INSERT(void* file, void* key, NUMBER size,NUMBER buffer,NUMBER bufsize)
LOCAL res,res2
res = pblKfInsert(file, key,size,(void*)buffer,bufsize)
IF res <> 0 THEN
res2 = -pbl_errno
RETURN res2
ELSE
RETURN res
ENDIF
END FUNCTION
'
' KEY_FIND
'
FUNCTION KEY_FIND(void* file, int mode, void* skey, NUMBER skeylen, void* okey, NUMBER oke)
LOCAL res2,res
res = pblKfFind(file, mode,skey,skeylen,okey,(size_t*)oke)
IF res < 0 THEN
res2 = -pbl_errno
RETURN res2
ELSE
RETURN res
ENDIF
END FUNCTION
'
' KEY_READ
'
FUNCTION KEY_READ(void* file, NUMBER buffer,NUMBER bufsize)
LOCAL res2,res
res = pblKfRead(file,(void*)buffer,bufsize)
IF res < 0 THEN
res2 = -pbl_errno
RETURN res2
ELSE
RETURN res
ENDIF
END FUNCTION
'
' KEY_COMMIT
'
'
' KEY_DELETE
'
'
' KEY_FLUSH
'
' KEY_SAVEP
'
'
' KEY_RESTP
'
'
Attachments:
