Working with curl

Edit: I retracted my below questions and wall of text. I leave this as a basic curl usage example.

Edit2: Updated example. Removed cascade of if/else. Reduced lines. More efficient and reusable.

Edit3: Found that since the callback is called from a c procedure, that using c calling convention is a better practice for the callbacks. Not doing that worked in this case, but when I tried more complicated things, it got weird. Added struct to carry data and context into the callback.

Edit4: Expanded example cause I can’t stop faffing with stuff. Sorry. More usage examples. Renamed stuff.

The libcurl.lib file needed for Windows is already in the vendor/curl/lib folder, so should compile straight-away.

On Linux, I had to add the following to compile:

sudo apt install libcurl4-gnutls-dev
sudo apt install libcurl4-openssl-dev
sudo apt install libmbedtls-dev
sudo apt install libz-dev

Example getinmemory from lib_curl examples.

package getinmemory

import "core:os"
import "core:fmt"
import "core:bytes"
import "vendor:curl"

CB_Write_Data :: struct {
	data: ^[dynamic]byte,
	ctx:  ^runtime.Context,
}

cb_write_data_proc :: proc"c"(ndata: rawptr, nsize: uint, nlength: uint, cb_write_data: rawptr) -> uint {
	wdata := (^CB_Write_Data)(cb_write_data) // convert CB_Write_Data rawptr
	nsize := nsize * nlength                 // new data to write
	wsize := uint(len(wdata.data))           // currently stored callback data
	context = wdata.ctx^                     // apply cloned context
	if resize_dynamic_array(wdata.data, wsize + nsize) != nil { return 0 }
	return uint(copy(wdata.data[wsize:], ([^]byte)(ndata)[:nsize]))
}

curl_http :: proc(url: cstring, data: ^[dynamic]byte, progress := false) -> (code: curl.code) {
	hcurl := curl.easy_init()
	if hcurl == nil { return .E_FAILED_INIT }
	defer curl.easy_cleanup(hcurl)

	ctx_clone, ctx_err := new_clone(context)
	if ctx_err != nil { return .E_FAILED_INIT }
	defer free(ctx_clone)
	
	wdata := CB_Write_Data{data, ctx_clone}
	curl.easy_setopt(hcurl, .URL, url) or_return
	curl.easy_setopt(hcurl, .WRITEFUNCTION, cb_write_data_proc) or_return
	curl.easy_setopt(hcurl, .WRITEDATA, &wdata) or_return
	curl.easy_setopt(hcurl, .USERAGENT, "libcurl-agent/1.0") or_return
	curl.easy_setopt(hcurl, .NOPROGRESS, progress ? 0:1) or_return
	curl.easy_perform(hcurl) or_return

	return .E_OK
}

main :: proc() {
	data := make([dynamic]byte, context.allocator)
	defer delete(data)

	url := cstring("https://www.example.com/") // use this for testing
	//url := cstring("https://wttr.in/Chicago-IL") // use this for funs
	//url := cstring("https://wttr.in/moon") // use this for more funs

	// global_init once for application and do multiple curl_http within

	// Get web data and print to terminal - progress disabled
	if curl.global_init(curl.GLOBAL_ALL) == .E_OK {
		defer curl.global_cleanup()
		if code := curl_http(url, &data, false); code != .E_OK {
			fmt.println("curl_http failed with code:", code)
		} else {
			fmt.printfln("%s", bytes.trim_right(data[:], {'\n', '\r'}))
		}
		clear(&data)
	}

	/*
	// Download a file and save to current directory - progress enabled
	if curl.global_init(curl.GLOBAL_ALL) == .E_OK {
		defer curl.global_cleanup()
		url_file := cstring("https://sample-files.com/downloads/compressed/zip/large-archive.zip")
		if code := curl_http(url_file, &data, true); code != .E_OK {
			fmt.println("curl_http failed with code:", code)
		} else if file, err := os.open("large-archive.zip", {.Create, .Write}); err == nil {
			os.write(file, data[:])
			os.close(file)
		}
		clear(&data)
	}
	*/
}
1 Like

Cool!

Integrating curl with nbio would serve as a great example, pretty sure it’s possible with the associate_handle proc

References:

1 Like

Custom transfer progress callback example from progressfunc with a few sensible additions.

CB_Write_Data :: struct {
	data: ^[dynamic]byte,
	ctx:  ^runtime.Context,
}

cb_write_data_proc :: proc"c"(ndata: rawptr, nsize: uint, nlength: uint, cb_write_data: rawptr) -> uint {
	wdata := (^CB_Write_Data)(cb_write_data) // convert CB_Write_Data rawptr
	nsize := nsize * nlength                 // new data to write
	wsize := uint(len(wdata.data))           // currently stored callback data
	context = wdata.ctx^                     // apply cloned context
	if resize_dynamic_array(wdata.data, wsize + nsize) != nil { return 0 }
	return uint(copy(wdata.data[wsize:], ([^]byte)(ndata)[:nsize]))
}

CB_Xfer_Data :: struct {
	lasttm: i64,
	lastdl: i64,
	lastul: i64,
	hnd: ^curl.CURL,
	ctx: ^runtime.Context,
}

MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL :: 250_000_000 // in ns this equals 0.25 seconds
cb_xfer_data_proc :: proc"c"(cb_xfer_data: rawptr, dltotal, dlnow, ultotal, ulnow: i64) -> int {
	xdata  := (^CB_Xfer_Data)(cb_xfer_data)
	context = xdata.ctx^
	
	// use this to limit based on N seconds if needed
	// for total/remaining time or speed calculations possibly
	// set MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL constant accordingly
	//curtime: i64
	//curl.easy_getinfo(xdata.hnd, .TOTAL_TIME_T, &curtime)
	//if curtime - xdata.lasttm >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL {
	//	xdata.lasttm = curtime
	//}

	// print or do something with data
	if dltotal > 0 && xdata.lastdl != dlnow {
		xdata.lastdl = dlnow
		fmt.printfln("%-4s% 10i / %i %3i%s", "DL:", dlnow, dltotal, int((f32(dlnow)/f32(dltotal)) * 100), "%")
	}
	if ultotal > 0 && xdata.lastul != ulnow {
		xdata.lastul = ulnow
		fmt.printfln("%-4s% 10i / %i %3i%s", "UL:", ulnow, ultotal, int((f32(ulnow)/f32(ultotal)) * 100), "%")
	}
	return 0
}

curl_http_cbxfer :: proc(url: cstring, data: ^[dynamic]byte) -> (code: curl.code) {
	hcurl := curl.easy_init()
	if hcurl == nil { return .E_FAILED_INIT }
	defer curl.easy_cleanup(hcurl)

	ctx_clone, ctx_err := new_clone(context)
	if ctx_err != nil { return .E_FAILED_INIT }
	defer free(ctx_clone)
	
	wdata := CB_Write_Data{data, ctx_clone}
	xdata := CB_Xfer_Data{0, 0, 0, hcurl, ctx_clone}
	curl.easy_setopt(hcurl, .URL, url) or_return
	curl.easy_setopt(hcurl, .WRITEFUNCTION, cb_write_data_proc) or_return
	curl.easy_setopt(hcurl, .WRITEDATA, &wdata) or_return
	curl.easy_setopt(hcurl, .USERAGENT, "libcurl-agent/1.0") or_return
	curl.easy_setopt(hcurl, .XFERINFOFUNCTION, cb_xfer_data_proc) or_return
	curl.easy_setopt(hcurl, .XFERINFODATA, &xdata) or_return
	curl.easy_setopt(hcurl, .NOPROGRESS, 0) or_return
	curl.easy_perform(hcurl) or_return

	return .E_OK
}

main :: proc() {
	data := make([dynamic]byte, context.allocator)
	defer delete(data)

	// Download a file and save to current directory with custom progress callback
	if curl.global_init(curl.GLOBAL_ALL) == .E_OK {
		defer curl.global_cleanup()
		url_file := cstring("https://sample-files.com/downloads/compressed/zip/large-archive.zip")
		if code := curl_http_cbxfer(url_file, &data); code != .E_OK {
			fmt.println("curl_http failed with code:", code)
		} else if file, err := os.open("large-archive.zip", {.Create, .Write}); err == nil {
			os.write(file, data[:])
			os.close(file)
		}
		clear(&data)
	}
}
1 Like