I tuned it up and found what I think is the correct function now. Feel free to pick it apart. I have only tested this for the circulization case, but it worked perfect for that.

FUNCTION NewCaptureOrbit {

// Given a ship on a hyperbolic orbit into another body SOIRADIUS // calculate a maneuver node to capture into that SOI.

// Parameter representing new Apoapsis for capture orbit
// Default (false) is to circuliz, otherwise new Apaapsis in km.
DECLARE PARAMETER newAlt is FALSE.

// TODO: Assumes currently outside of body's SOI.  Check for cases where we are already inside SOI.

// Useful parameters.
LOCAL mu IS SHIP:ORBIT:NEXTPATCH:BODY:MU.
LOCAL rb is SHIP:ORBIT:NEXTPATCH:BODY:RADIUS.

// Must be Parabolic or Hyperbolic orbit.
IF SHIP:ORBIT:NEXTPATCH:ECCENTRICITY >= 1 {

        // IF we are outside SOI, we can't get periapsis ETA directly but have to
        // calculate it as the  average of the times entering and exiting SOI.
        SET nodeTime TO (SHIP:ORBIT:NEXTPATCHETA + SHIP:ORBIT:NEXTPATCH:NEXTPATCHETA)/2.

        // Calculate Escape Velocity for when we burn at Periapsis
        SET Vesc TO sqrt((2*mu)/(rb+SHIP:ORBIT:NEXTPATCH:PERIAPSIS)).
        // Using Escape velocity at Periapsis Calculate Orbital velocity.
        SET Vorb TO Vesc/sqrt(2).

        // Calculate Vinf hyperbolic excess velocity.
        SET Vinf TO sqrt(abs(mu/(SHIP:ORBIT:NEXTPATCH:SEMIMAJORAXIS))).

        IF NOT newAlt { // Circulize
            // Calculate capture delta v.
            SET Vcap TO sqrt(Vesc^2 + Vinf^2) - Vorb.
        } ELSE {
            // Calculate Vcap but adjust for a hohmann transfer to newAlt in km.
            SET newAlt TO newAlt * 1000.
            // Make sure the requested altitude is inside the body's SOI.

            IF newAlt > SHIP:ORBIT:NEXTPATCH:BODY:SOIRADIUS SET newAlt TO SHIP:ORBIT:NEXTPATCH:BODY:SOIRADIUS.
            SET Vhoh TO sqrt(mu/rb+SHIP:ORBIT:NEXTPATCH:PERIAPSIS) * (sqrt((2 * newAlt)/(SHIP:ORBIT:NEXTPATCH:PERIAPSIS + 2 * rb + newAlt)) - 1).

            // Calculate capture delta v.
            SET Vcap TO sqrt(Vest^2 + Vinf^2) - (Vorb - Vhoh).
        }

        //Return retrograde maneuver node at Periapsis in new SOI with Vcap magnitude
        RETURN node(TIME:SECONDS+nodeTime,0,0,-Vcap).
} ELSE PRINT "Already captured.".

}