This sequence of commands looks correct. You can simplify it by using shell redirection instead of temporary files:
base64 -d b64.txt | openssl dgst -sha1 -binary | base64
To do the same in C using the OpenSSL library, you can use abstraction BIOfor a good effect:
#include <stdio.h>
#include <openssl/bio.h>
#include <openssl/evp.h>
int main(int argc, char *argv[])
{
BIO *bio_in, *b64, *md, *bio_out;
char buf[1024];
char mdbuf[EVP_MAX_MD_SIZE];
int mdlen;
bio_in = BIO_new_fp(stdin, BIO_NOCLOSE);
b64 = BIO_new(BIO_f_base64());
bio_in = BIO_push(b64, bio_in);
md = BIO_new(BIO_f_md());
BIO_set_md(md, EVP_sha1());
bio_in = BIO_push(md, bio_in);
while (BIO_read(bio_in, buf, sizeof buf) > 0)
;
mdlen = BIO_gets(md, mdbuf, sizeof mdbuf);
bio_out = BIO_new_fp(stdout, BIO_NOCLOSE);
b64 = BIO_new(BIO_f_base64());
bio_out = BIO_push(b64, bio_out);
BIO_write(bio_out, mdbuf, mdlen);
BIO_flush(bio_out);
BIO_free_all(bio_in);
BIO_free_all(bio_out);
return 0;
}
base64 stdin SHA1 base64 stdout.