Did you use BASS_Encode_Write to send the first block of data to the encoder? If not, doing that will hopefully fix the problem. Something like this:
BOOL CALLBACK RecordProc(HRECORD handle, const void *buffer, DWORD length, void *user)
{
if (!encoder) { // haven't started encoding yet
float level;
BASS_ChannelGetLevelEx(handle, &level, 0.1, BASS_LEVEL_MONO); // get the level
if (level >= threshold) { // hit the threshold
encoder = BASS_Encode_Start(...); // start the encoder
BASS_Encode_Write(encoder, buffer, length); // send this block to it
}
}
return true;
}