The indentation suggests the complex type is being defined within the EnrolledCourse element declaration. The line containing 
<xs:complexType name="StudentType"> 
should be at the same indentation level with the element definition

<xs:element name="EnrolledCourse" type="sml:refType"
                sml:targetType="tns:CourseType"/>
    <xs:complexType name="StudentType">     <<<<<<<<<<<<<<<<<<<<<<<<<<<<<
        <xs:sequence>
            <xs:element name="ID" type="xs:string"/>
            <xs:element name="Name" type="xs:string"/>
            <xs:element name="EnrolledCourses" minOccurs="0">
                <xs:complexType>
                    <xs:sequence>
                        <xs:element ref="tns:EnrolledCourse"
                                    maxOccurs="unbounded"/>
                    </xs:sequence>
                </xs:complexType>
            </xs:element>
        </xs:sequence>
</xs:complexType>