Isn't this faster?
SELECTPatient_ID,
COUNT(*) AS Number_Of_Visits
FROM(
SELECT DISTINCTPatient_ID,
DATEDIFF(DAY, '19000101', Er_Date) - DENSE_RANK() OVER (PARTITION BY Patient_ID ORDER BY DATEDIFF(DAY, '19000101', Er_Date)) AS DayKey
FROM@TempTable
) AS d
GROUP BYPatient_ID
ORDER BYPatient_ID
N 56°04'39.16"
E 12°55'05.25"